ha/service-mgmt/sm-1.0.0/src/sm_service_group_fsm.c

1291 lines
48 KiB
C

//
// Copyright (c) 2014 Wind River Systems, Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
#include "sm_service_group_fsm.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "sm_types.h"
#include "sm_debug.h"
#include "sm_time.h"
#include "sm_list.h"
#include "sm_selobj.h"
#include "sm_service_group_table.h"
#include "sm_service_group_member_table.h"
#include "sm_service_table.h"
#include "sm_service_group_engine.h"
#include "sm_service_group_initial_state.h"
#include "sm_service_group_active_state.h"
#include "sm_service_group_go_standby_state.h"
#include "sm_service_group_go_active_state.h"
#include "sm_service_group_standby_state.h"
#include "sm_service_group_disabling_state.h"
#include "sm_service_group_disabled_state.h"
#include "sm_service_group_shutdown_state.h"
#include "sm_service_group_enable.h"
#include "sm_service_group_go_active.h"
#include "sm_service_group_go_standby.h"
#include "sm_service_group_disable.h"
#include "sm_service_group_audit.h"
#include "sm_service_api.h"
#include "sm_log.h"
#include "sm_node_utils.h"
#include "sm_node_swact_monitor.h"
#include "sm_failover.h"
#include "sm_swact_state.h"
static SmListT* _callbacks = NULL;
// ****************************************************************************
// Service Group FSM - Register Callback
// =====================================
SmErrorT sm_service_group_fsm_register_callback(
SmServiceGroupFsmCallbackT callback )
{
SM_LIST_PREPEND( _callbacks, (SmListEntryDataPtrT) callback );
return( SM_OKAY );
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Deregister Callback
// =======================================
SmErrorT sm_service_group_fsm_deregister_callback(
SmServiceGroupFsmCallbackT callback )
{
SM_LIST_REMOVE( _callbacks, (SmListEntryDataPtrT) callback );
return( SM_OKAY );
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Notify
// ==========================
static void sm_service_group_fsm_notify( SmServiceGroupT* service_group,
const char reason_text[] )
{
SmListT* entry = NULL;
SmListEntryDataPtrT entry_data;
SmServiceGroupFsmCallbackT callback;
SM_LIST_FOREACH( _callbacks, entry, entry_data )
{
callback = (SmServiceGroupFsmCallbackT) entry_data;
callback( service_group->name, service_group->state,
service_group->status, service_group->condition,
service_group->health, reason_text );
}
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Update Service Group Member
// ===============================================
static void sm_service_group_fsm_update_service_group_member(
void* user_data[], SmServiceGroupMemberT* service_group_member )
{
SmServiceT* service;
service = sm_service_table_read( service_group_member->service_name );
if( NULL == service )
{
DPRINTFE( "Failed to read service group member (%s), error=%s.",
service_group_member->service_name,
sm_error_str(SM_NOT_FOUND) );
return;
}
if(( service->state != service_group_member->service_state )||
( service->status != service_group_member->service_status ))
{
service_group_member->service_state = service->state;
service_group_member->service_status = service->status;
}
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Update Service Group Members
// ================================================
static SmErrorT sm_service_group_fsm_update_service_group_members( void )
{
sm_service_group_member_table_foreach( NULL,
sm_service_group_fsm_update_service_group_member );
return( SM_OKAY );
}
// ****************************************************************************
static void sm_service_group_state_check( void* user_data[],
SmServiceGroupT* service_group )
{
bool *all_good = (bool*)user_data[0];
if( service_group->desired_state != service_group->state )
{
*all_good = false;
}
}
// ****************************************************************************
// Service Group FSM - Enter State
// ===============================
static SmErrorT sm_service_group_fsm_enter_state( SmServiceGroupT* service_group )
{
SmErrorT error;
switch( service_group->state )
{
case SM_SERVICE_GROUP_STATE_INITIAL:
error = sm_service_group_initial_state_entry( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to enter state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_ACTIVE:
error = sm_service_group_active_state_entry( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to enter state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_GO_ACTIVE:
error = sm_service_group_go_active_state_entry( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to enter state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_GO_STANDBY:
error = sm_service_group_go_standby_state_entry( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to enter state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_STANDBY:
error = sm_service_group_standby_state_entry( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to enter state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_DISABLING:
error = sm_service_group_disabling_state_entry( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to enter state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_DISABLED:
// This is a workaround for lack of sm message that
// signifies the "end-of-swact" state on both controller nodes.
//
// As all service groups have to transition to disabled state
// during swact regardless of the outcome (success or failure), it
// is a suitable spot to reaffine tasks back to the platform cores.
// Controller services group is often the last group to reach
// any state. Task reaffining is only applicable to AIO duplex.
bool duplex;
sm_node_utils_is_aio_duplex(&duplex);
if (( duplex ) && ( 0 == strcmp( "controller-services", service_group->name )))
{
DPRINTFI( "Reaffining tasks back to platform cores..." );
sm_set_swact_state(SM_SWACT_STATE_END);
}
error = sm_service_group_disabled_state_entry( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to enter state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_SHUTDOWN:
error = sm_service_group_shutdown_state_entry( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to enter state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
default:
DPRINTFE( "Unknown service group (%s) state (%s).",
service_group->name,
sm_service_group_state_str( service_group->state ) );
break;
}
if( SM_SERVICE_GROUP_STATE_STANDBY == service_group->state ||
SM_SERVICE_GROUP_STATE_ACTIVE == service_group->state )
{
bool all_good = true;
void* user_data[] = {&all_good};
sm_service_group_table_foreach( user_data, sm_service_group_state_check );
if( all_good )
{
SmNodeScheduleStateT controller_state = get_controller_state();
char hostname[SM_NODE_NAME_MAX_CHAR];
error = sm_node_utils_get_hostname( hostname );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to get hostname, error=%s.",
sm_error_str( error ) );
hostname[0] = '\0';
}
if( SM_NODE_STATE_ACTIVE == controller_state )
{
SmNodeSwactMonitor::SwactUpdate(hostname, SM_NODE_STATE_ACTIVE );
}
else if ( SM_NODE_STATE_STANDBY == controller_state )
{
SmNodeSwactMonitor::SwactUpdate(hostname, SM_NODE_STATE_STANDBY );
}else
{
SmNodeSwactMonitor::SwactUpdate(hostname, SM_NODE_STATE_FAILED );
}
}
}
return( SM_OKAY );
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Exit State
// ==============================
static SmErrorT sm_service_group_fsm_exit_state( SmServiceGroupT* service_group )
{
SmErrorT error;
switch( service_group->state )
{
case SM_SERVICE_GROUP_STATE_INITIAL:
error = sm_service_group_initial_state_exit( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to exit state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_ACTIVE:
error = sm_service_group_active_state_exit( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to exit state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_GO_ACTIVE:
error = sm_service_group_go_active_state_exit( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to exit state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_GO_STANDBY:
error = sm_service_group_go_standby_state_exit( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to exit state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_STANDBY:
error = sm_service_group_standby_state_exit( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to exit state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_DISABLING:
error = sm_service_group_disabling_state_exit( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to exit state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_DISABLED:
error = sm_service_group_disabled_state_exit( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to exit state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_SHUTDOWN:
error = sm_service_group_shutdown_state_exit( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to exit state (%s), "
"error=%s.", service_group->name,
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
default:
DPRINTFE( "Unknown service group (%s) state (%s).",
service_group->name,
sm_service_group_state_str( service_group->state ) );
break;
}
return( SM_OKAY );
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Transition State
// ====================================
static SmErrorT sm_service_group_fsm_transition_state(
SmServiceGroupT* service_group, SmServiceGroupStateT from_state )
{
SmErrorT error;
switch( service_group->state )
{
case SM_SERVICE_GROUP_STATE_INITIAL:
error = sm_service_group_initial_state_transition( service_group,
from_state );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to transition from "
"state (%s) to state (%s), error=%s.",
service_group->name,
sm_service_group_state_str( from_state ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_ACTIVE:
error = sm_service_group_active_state_transition( service_group,
from_state );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to transition from "
"state (%s) to state (%s), error=%s.",
service_group->name,
sm_service_group_state_str( from_state ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_GO_ACTIVE:
error = sm_service_group_go_active_state_transition( service_group,
from_state );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to transition from "
"state (%s) to state (%s), error=%s.",
service_group->name,
sm_service_group_state_str( from_state ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_GO_STANDBY:
error = sm_service_group_go_standby_state_transition( service_group,
from_state );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to transition from "
"state (%s) to state (%s), error=%s.",
service_group->name,
sm_service_group_state_str( from_state ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_STANDBY:
error = sm_service_group_standby_state_transition( service_group,
from_state );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to transition from "
"state (%s) to state (%s), error=%s.",
service_group->name,
sm_service_group_state_str( from_state ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_DISABLING:
error = sm_service_group_disabling_state_transition( service_group,
from_state );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to transition from "
"state (%s) to state (%s), error=%s.",
service_group->name,
sm_service_group_state_str( from_state ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_DISABLED:
error = sm_service_group_disabled_state_transition( service_group,
from_state );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to transition from "
"state (%s) to state (%s), error=%s.",
service_group->name,
sm_service_group_state_str( from_state ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_SHUTDOWN:
error = sm_service_group_shutdown_state_transition( service_group,
from_state );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to transition from "
"state (%s) to state (%s), error=%s.",
service_group->name,
sm_service_group_state_str( from_state ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
default:
DPRINTFE( "Unknown service group (%s) state (%s).",
service_group->name,
sm_service_group_state_str( service_group->state ) );
break;
}
return( SM_OKAY );
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Set State
// =============================
SmErrorT sm_service_group_fsm_set_state( char service_group_name[],
SmServiceGroupStateT state )
{
SmServiceGroupStateT prev_state;
SmServiceGroupT* service_group;
SmErrorT error, error2;
service_group = sm_service_group_table_read( service_group_name );
if( NULL == service_group )
{
DPRINTFE( "Failed to read service group (%s), error=%s.",
service_group_name, sm_error_str(SM_NOT_FOUND) );
return( SM_NOT_FOUND );
}
prev_state = service_group->state;
error = sm_service_group_fsm_exit_state( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to exit state (%s) service group (%s), error=%s.",
sm_service_group_state_str( service_group->state ),
service_group_name, sm_error_str( error ) );
return( error );
}
service_group->state = state;
error = sm_service_group_fsm_transition_state( service_group,
prev_state );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to transition to state (%s) service group (%s), "
"error=%s.", sm_service_group_state_str( service_group->state ),
service_group_name, sm_error_str( error ) );
goto STATE_CHANGE_ERROR;
}
error = sm_service_group_fsm_enter_state( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to enter state (%s) service group (%s), error=%s.",
sm_service_group_state_str( service_group->state ),
service_group_name, sm_error_str( error ) );
goto STATE_CHANGE_ERROR;
}
return( SM_OKAY );
STATE_CHANGE_ERROR:
error2 = sm_service_group_fsm_exit_state( service_group );
if( SM_OKAY != error2 )
{
DPRINTFE( "Failed to exit state (%s) service group (%s), error=%s.",
sm_service_group_state_str( service_group->state ),
service_group_name, sm_error_str( error2 ) );
abort();
}
service_group->state = prev_state;
error2 = sm_service_group_fsm_transition_state( service_group, state );
if( SM_OKAY != error2 )
{
DPRINTFE( "Failed to transition to state (%s) service group (%s), "
"error=%s.", sm_service_group_state_str( service_group->state ),
service_group_name, sm_error_str( error2 ) );
abort();
}
error2 = sm_service_group_fsm_enter_state( service_group );
if( SM_OKAY != error2 )
{
DPRINTFE( "Failed to enter state (%s) service group (%s), error=%s.",
sm_service_group_state_str( service_group->state ),
service_group_name, sm_error_str( error2 ) );
abort();
}
return( error );
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Get Next State
// ==================================
SmErrorT sm_service_group_fsm_get_next_state( char service_group_name[],
SmServiceGroupEventT event, SmServiceGroupStateT* state )
{
bool complete;
SmServiceGroupT* service_group;
SmErrorT error;
service_group = sm_service_group_table_read( service_group_name );
if( NULL == service_group )
{
DPRINTFE( "Failed to read service group (%s), error=%s.",
service_group_name, sm_error_str(SM_NOT_FOUND) );
return( SM_NOT_FOUND );
}
error = sm_service_group_fsm_update_service_group_members();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to update service group (%s) members, "
"error=%s.", service_group->name,
sm_error_str( error ) );
return( error );
}
switch( event )
{
case SM_SERVICE_GROUP_EVENT_GO_ACTIVE:
error = sm_service_group_go_active_complete( service_group,
&complete );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to determine if go-active complete on "
"service group (%s), error=%s",
service_group->name, sm_error_str( error ) );
return( error );
}
if( complete )
{
*state = SM_SERVICE_GROUP_STATE_ACTIVE;
} else {
*state = SM_SERVICE_GROUP_STATE_GO_ACTIVE;
}
break;
case SM_SERVICE_GROUP_EVENT_GO_STANDBY:
error = sm_service_group_go_standby_complete( service_group,
&complete );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to determine if go-standby complete on "
"service group (%s), error=%s",
service_group->name, sm_error_str( error ) );
return( error );
}
if( complete )
{
*state = SM_SERVICE_GROUP_STATE_STANDBY;
} else {
*state = SM_SERVICE_GROUP_STATE_GO_STANDBY;
}
break;
case SM_SERVICE_GROUP_EVENT_DISABLE:
error = sm_service_group_disable_complete( service_group,
&complete );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to determine if disable complete on "
"service group (%s), error=%s",
service_group->name, sm_error_str( error ) );
return( error );
}
if( complete )
{
*state = SM_SERVICE_GROUP_STATE_DISABLED;
} else {
*state = SM_SERVICE_GROUP_STATE_DISABLING;
}
break;
default:
// Ignore.
break;
}
return( SM_OKAY );
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Event Handler
// =================================
SmErrorT sm_service_group_fsm_event_handler( char service_group_name[],
SmServiceGroupEventT event, void* event_data[], const char reason_text[] )
{
SmServiceGroupStateT prev_state;
SmServiceGroupStatusT prev_status;
SmServiceGroupConditionT prev_condition;
char prev_reason_text[SM_SERVICE_GROUP_REASON_TEXT_MAX_CHAR];
SmServiceGroupT* service_group;
SmErrorT error;
service_group = sm_service_group_table_read( service_group_name );
if( NULL == service_group )
{
DPRINTFE( "Failed to read service group (%s), error=%s.",
service_group_name, sm_error_str(SM_NOT_FOUND) );
return( SM_NOT_FOUND );
}
prev_state = service_group->state;
prev_status = service_group->status;
prev_condition = service_group->condition;
snprintf( prev_reason_text, sizeof(prev_reason_text), "%s",
service_group->reason_text );
switch( service_group->state )
{
case SM_SERVICE_GROUP_STATE_INITIAL:
error = sm_service_group_initial_state_event_handler(
service_group, event, event_data );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to handle event (%s) "
"in state (%s), error=%s.", service_group_name,
sm_service_group_event_str( event ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_ACTIVE:
error = sm_service_group_active_state_event_handler(
service_group, event, event_data );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to handle event (%s) "
"in state (%s), error=%s.", service_group_name,
sm_service_group_event_str( event ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_GO_ACTIVE:
error = sm_service_group_go_active_state_event_handler(
service_group, event, event_data );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to handle event (%s) "
"in state (%s), error=%s.", service_group_name,
sm_service_group_event_str( event ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_GO_STANDBY:
error = sm_service_group_go_standby_state_event_handler(
service_group, event, event_data );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to handle event (%s) "
"in state (%s), error=%s.", service_group_name,
sm_service_group_event_str( event ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_STANDBY:
error = sm_service_group_standby_state_event_handler(
service_group, event, event_data );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to handle event (%s) "
"in state (%s), error=%s.", service_group_name,
sm_service_group_event_str( event ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_DISABLING:
error = sm_service_group_disabling_state_event_handler(
service_group, event, event_data );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to handle event (%s) "
"in state (%s), error=%s.", service_group_name,
sm_service_group_event_str( event ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_DISABLED:
error = sm_service_group_disabled_state_event_handler(
service_group, event, event_data );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to handle event (%s) "
"in state (%s), error=%s.", service_group_name,
sm_service_group_event_str( event ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
case SM_SERVICE_GROUP_STATE_SHUTDOWN:
error = sm_service_group_shutdown_state_event_handler(
service_group, event, event_data );
if( SM_OKAY != error )
{
DPRINTFE( "Service group (%s) unable to handle event (%s) "
"in state (%s), error=%s.", service_group_name,
sm_service_group_event_str( event ),
sm_service_group_state_str( service_group->state ),
sm_error_str( error ) );
return( error );
}
break;
default:
DPRINTFE( "Unknown state (%s) for service group (%s).",
sm_service_group_state_str( service_group->state ),
service_group_name );
break;
}
if(( prev_state != service_group->state )||
( prev_status != service_group->status )||
( prev_condition != service_group->condition ))
{
DPRINTFI( "Service group (%s) was in the %s%s%s state and is now "
"in the %s%s%s state%s%s.",
service_group_name,
sm_service_group_state_str( prev_state ),
SM_SERVICE_GROUP_STATUS_NONE == prev_status
? "" : "-",
SM_SERVICE_GROUP_STATUS_NONE == prev_status
? "" : sm_service_group_status_str(prev_status),
sm_service_group_state_str( service_group->state ),
SM_SERVICE_GROUP_STATUS_NONE == service_group->status
? "" : "-",
SM_SERVICE_GROUP_STATUS_NONE == service_group->status
? "" : sm_service_group_status_str(service_group->status),
SM_SERVICE_GROUP_CONDITION_NONE == service_group->condition
? "" : ", condition=",
SM_SERVICE_GROUP_CONDITION_NONE == service_group->condition
? "" : sm_service_group_condition_str( service_group->condition ) );
error = sm_service_group_table_persist( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to persist service group (%s) data, error=%s.",
service_group->name, sm_error_str(error) );
return( error );
}
if(( prev_state != service_group->state )||
( prev_status != service_group->status ))
{
sm_log_service_group_state_change( service_group_name,
sm_service_group_state_str( prev_state ),
sm_service_group_status_str( prev_status ),
sm_service_group_state_str( service_group->state ),
sm_service_group_status_str( service_group->status ),
service_group->reason_text );
}
sm_service_group_fsm_notify( service_group,
service_group->reason_text );
} else if( SM_SERVICE_GROUP_EVENT_AUDIT == event ) {
if(( SM_SERVICE_GROUP_STATE_ACTIVE == service_group->state )||
( SM_SERVICE_GROUP_STATE_STANDBY == service_group->state )||
( SM_SERVICE_GROUP_STATE_DISABLED == service_group->state ))
{
sm_service_group_fsm_notify( service_group,
service_group->reason_text );
}
} else if( 0 != strcmp( prev_reason_text, service_group->reason_text ) ) {
sm_service_group_fsm_notify( service_group,
service_group->reason_text );
}
if( service_group->desired_state != service_group->state )
{
error = sm_service_group_engine_signal( service_group );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to signal service group (%s), error=%s.",
service_group_name, sm_error_str( error ) );
return( error );
}
}
return( SM_OKAY );
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Recover Member
// ==================================
static void sm_service_group_fsm_recover_member( void* user_data[],
SmServiceGroupMemberT* service_group_member )
{
bool clear_fatal_condition = *(bool*) user_data[0];
SmErrorT error;
error = sm_service_api_recover( service_group_member->service_name,
clear_fatal_condition );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to recover member (%s), error=%s.",
service_group_member->service_name,
sm_error_str( error ) );
return;
}
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Recover
// ===========================
SmErrorT sm_service_group_fsm_recover( char service_group_name[],
bool clear_fatal_condition )
{
void* user_data[] = {&clear_fatal_condition};
sm_service_group_member_table_foreach_member( service_group_name, user_data,
sm_service_group_fsm_recover_member);
return( SM_OKAY );
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Service State Change Notification
// =====================================================
static void sm_service_group_fsm_service_scn( char service_name[],
SmServiceStateT state, SmServiceStatusT status,
SmServiceConditionT condition )
{
char reason_text[SM_LOG_REASON_TEXT_MAX_CHAR] = "";
SmServiceGroupMemberT* service_group_member;
void* event_data[SM_SERVICE_GROUP_EVENT_DATA_MAX] = {0};
SmErrorT error;
event_data[SM_SERVICE_GROUP_EVENT_DATA_SERVICE_NAME] = service_name;
event_data[SM_SERVICE_GROUP_EVENT_DATA_SERVICE_STATE] = &state;
service_group_member = sm_service_group_member_table_read_by_service(
service_name );
if( NULL == service_group_member )
{
DPRINTFE( "Failed to query service group member by service (%s), "
"error=%s.", service_name, sm_error_str(SM_NOT_FOUND) );
return;
}
service_group_member->service_state = state;
service_group_member->service_status = status;
service_group_member->service_condition = condition;
if( SM_SERVICE_STATUS_FAILED == status )
{
service_group_member->service_failure_timestamp
= sm_time_get_elapsed_ms( NULL );
}
if( SM_SERVICE_STATUS_NONE == status )
{
snprintf( reason_text, sizeof(reason_text), "%s service state "
"change, state=%s", service_name, sm_service_state_str(state) );
} else {
if( SM_SERVICE_CONDITION_NONE == condition )
{
snprintf( reason_text, sizeof(reason_text), "%s service state "
"change, state=%s, status=%s", service_name,
sm_service_state_str(state),
sm_service_status_str(status) );
} else {
snprintf( reason_text, sizeof(reason_text), "%s service state "
"change, state=%s, status=%s, condition=%s",
service_name, sm_service_state_str(state),
sm_service_status_str(status),
sm_service_condition_str(condition) );
}
}
error = sm_service_group_fsm_event_handler( service_group_member->name,
SM_SERVICE_GROUP_EVENT_SERVICE_SCN,
event_data, reason_text );
if( SM_OKAY != error )
{
DPRINTFE( "Service (%s) scn not handled for service group (%s), "
"error=%s.", service_group_member->service_name,
service_group_member->name, sm_error_str( error ) );
return;
}
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Initialize
// ==============================
SmErrorT sm_service_group_fsm_initialize( void )
{
SmErrorT error;
error = sm_service_group_initial_state_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group initial state module, "
"error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_active_state_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group active state "
"module, error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_go_active_state_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group go-active state "
"module, error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_go_standby_state_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group go-standby "
"state module, error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_standby_state_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group standby state "
"module, error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_disabling_state_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group disabling state "
"module, error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_disabled_state_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group disabled state "
"module, error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_shutdown_state_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group shutdown state "
"module, error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_enable_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group enable module, "
"error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_go_active_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group go-active module, "
"error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_go_standby_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group go-standby module, "
"error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_disable_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group disable module, "
"error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_audit_initialize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to initialize service group audit module, "
"error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_api_register_callback( sm_service_group_fsm_service_scn );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to register for service state changes, "
"error=%s.", sm_error_str( error ) );
return( error );
}
return( SM_OKAY );
}
// ****************************************************************************
// ****************************************************************************
// Service Group FSM - Finalize
// ============================
SmErrorT sm_service_group_fsm_finalize( void )
{
SmErrorT error;
error = sm_service_api_deregister_callback( sm_service_group_fsm_service_scn );
if( SM_OKAY != error )
{
DPRINTFE( "Failed to deregister for service state changes, "
"error=%s.", sm_error_str( error ) );
return( error );
}
error = sm_service_group_initial_state_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group initial state module, "
"error=%s.", sm_error_str( error ) );
}
error = sm_service_group_active_state_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group active state "
"module, error=%s.", sm_error_str( error ) );
}
error = sm_service_group_go_active_state_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group go-active state "
"module, error=%s.", sm_error_str( error ) );
}
error = sm_service_group_go_standby_state_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group go-standby state "
"module, error=%s.", sm_error_str( error ) );
}
error = sm_service_group_standby_state_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group standby state "
"module, error=%s.", sm_error_str( error ) );
}
error = sm_service_group_disabling_state_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group disabling state "
"module, error=%s.", sm_error_str( error ) );
}
error = sm_service_group_disabled_state_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group disabled state "
"module, error=%s.", sm_error_str( error ) );
}
error = sm_service_group_shutdown_state_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group shutdown state "
"module, error=%s.", sm_error_str( error ) );
}
error = sm_service_group_enable_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group enable module, "
"error=%s.", sm_error_str( error ) );
}
error = sm_service_group_go_active_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group go-active module, "
"error=%s.", sm_error_str( error ) );
}
error = sm_service_group_go_standby_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group go-standby module, "
"error=%s.", sm_error_str( error ) );
}
error = sm_service_group_disable_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group disable module, "
"error=%s.", sm_error_str( error ) );
}
error = sm_service_group_audit_finalize();
if( SM_OKAY != error )
{
DPRINTFE( "Failed to finalize service group audit module, "
"error=%s.", sm_error_str( error ) );
}
return( SM_OKAY );
}
// ****************************************************************************