From db1eea124dc41e0be3a7950a27ddb0bfe50b9c9a Mon Sep 17 00:00:00 2001 From: Steven Webster Date: Thu, 15 Dec 2022 16:46:11 +0000 Subject: [PATCH] Add admin network support to SM Add SM support for the DC admin network This commit adds SM support for the DC admin network. The admin network is intended to be used between a subcloud and system controller. Because the (existing) management network is so embedded in other parts of the StarlingX system, it makes it prohibitively hard to re-configure this network after initial installation. The admin network is intended to be isolated from the management network, allowing re-configuration of the network parameters in the case that the physical network between subcloud and system controller has been changed. In the case of admin network usage, the management network still exist but is a private network in the context of a subcloud. This specific commit provides for admin-ip and admin-interface services to be added to the SM database and be recognized in processing similar to the management, cluster-host, oam, etc networks. Since there is a requirement for the admin IP subnet information to be allowed to change at runtime, in-service updating of SM information relating to the admin-ip service (floating IP), as well as unicast heartbeating between peers is also added in this commit. Testing: AIO-SX: - admin-ip service is enabled when the admin network is created. - admin-ip service is not enabled when the admin network is not created. - floating-ip is updated on the admin interface when admin addr-pool information is changed. AIO-DX: - admin-ip service is enabled when the admin network is created. - admin-ip service is not enabled when the admin network is not created. - floating-ip is updated on the active-controller when the admin addr-pool information is changed. - When a peer admin interface is down, an alarm is raised. - When a peer admin IP is not correct (changed), an alarm is raised. - Swact between controllers. - Inactive controller admin interface goes down Result: A 400.005 major communication loss fault is generated for the inactive controller entity - Inactive controller admin interface comes back up Result: The fault is cleared - Inactive controller admin IP address is removed/changed Result: Two 400.005 major communication loss faults are generated for both controller entities - Inactive controller admin node IP address is re-applied Result: The faults are cleared - Active admin interface goes down Result: A 400.005 major communication loss fault is generated for the inactive controller entity. A swact is not issued. - Active admin interface comes back up Result: The fault is cleared - Active admin node IP address is removed/changed Result: Two 400.005 major communication loss faults are generated for both controller entities. A swact is not issued. - Active admin floating IP address is removed/changed Result: A 400.001 critical admin-services / admin-ip alarm is raised. A swact occurs. The floating admin IP is applied to the newly active controller. Alarms are cleared. - After the above test, the newly active controller swacts back to the previously active controller. Result: No alarms. The floating IP is applied to the newly active controller. - The cable for the management interface on the active controller is pulled Result: A swact occurs - The cable for the OAM interface on the active controller is pulled Result: A swact occurs - The cable for the Admin interface on the active controller is pulled Result: A swact occurs. 400.005 alarms are raised. - The mgmt, cluster-host, oam interfaces are all brought down/up at the same time. The admin interface is also brought down, but not brought back up back up. Result: A swact occurs, with multiple controller-services related to the mgmt interface being in degraded state. Story: 2010319 Task: 47278 Signed-off-by: Steven Webster Change-Id: I65df52600f4d5c499dceed32739cab414d36847a --- .../sm-tools/sm_tools/sm_api_msg_utils.py | 41 ++++++++- .../sm-tools/sm_tools/sm_configure.py | 48 ++++++++++- .../sm-tools/sm_tools/sm_provision.py | 33 ++++--- service-mgmt/sm-common/src/sm_eru_process.c | 9 +- service-mgmt/sm-common/src/sm_node_utils.c | 13 ++- service-mgmt/sm-common/src/sm_node_utils.h | 8 +- service-mgmt/sm-common/src/sm_types.c | 5 +- service-mgmt/sm-common/src/sm_types.h | 7 +- service-mgmt/sm-db/database/create_sm_db.sql | 9 ++ service-mgmt/sm/src/sm_api.c | 78 ++++++++++++++++- service-mgmt/sm/src/sm_api.h | 13 ++- service-mgmt/sm/src/sm_configure.cpp | 85 ++++++++++++++++++- service-mgmt/sm/src/sm_configure.h | 14 ++- service-mgmt/sm/src/sm_failover.c | 56 +++++++++++- service-mgmt/sm/src/sm_failover.h | 3 +- .../sm/src/sm_failover_fail_pending_state.cpp | 3 +- .../sm/src/sm_failover_failed_state.cpp | 19 ++++- service-mgmt/sm/src/sm_failover_fsm.cpp | 7 +- service-mgmt/sm/src/sm_failover_fsm.h | 3 +- .../sm/src/sm_failover_normal_state.cpp | 8 +- service-mgmt/sm/src/sm_failover_ss.c | 10 ++- service-mgmt/sm/src/sm_failover_ss.h | 3 +- service-mgmt/sm/src/sm_heartbeat_thread.c | 19 +++-- service-mgmt/sm/src/sm_main_event_handler.c | 26 +++++- .../sm/src/sm_service_domain_interface_api.c | 61 ++++++++++++- .../sm/src/sm_service_domain_interface_api.h | 13 +++ ...ervice_domain_interface_not_in_use_state.c | 38 +++++++++ 27 files changed, 585 insertions(+), 47 deletions(-) diff --git a/service-mgmt-tools/sm-tools/sm_tools/sm_api_msg_utils.py b/service-mgmt-tools/sm-tools/sm_tools/sm_api_msg_utils.py index ebd3962a..08316223 100644 --- a/service-mgmt-tools/sm-tools/sm_tools/sm_api_msg_utils.py +++ b/service-mgmt-tools/sm-tools/sm_tools/sm_api_msg_utils.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016 Wind River Systems, Inc. +# Copyright (c) 2016-2023 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -23,6 +23,9 @@ SM_API_MSG_SKIP_DEP_CHECK = "skip-dep" SM_API_MSG_TYPE_PROVISION_SERVICE = "PROVISION_SERVICE" SM_API_MSG_TYPE_DEPROVISION_SERVICE = "DEPROVISION_SERVICE" +SM_API_MSG_TYPE_PROVISION_SERVICE_DOMAIN_INTERFACE = "PROVISION_SERVICE_DOMAIN_INTERFACE" +SM_API_MSG_TYPE_DEPROVISION_SERVICE_DOMAIN_INTERFACE = "DEPROVISION_SERVICE_DOMAIN_INTERFACE" + SM_API_MSG_TYPE_RELOAD_DATA = "RELOAD_DATA" SM_API_MSG_TYPE_SDI_SET_STATE = "SERVICE_DOMAIN_INTERFACE_SET_STATE" @@ -34,6 +37,10 @@ SM_API_MSG_SEQNO_FIELD = 2 SM_API_MSG_TYPE_FIELD = 3 SM_API_MSG_ORIGIN_FIELD = 4 SM_API_MSG_SERVICE_NAME_FIELD = 5 +# For provisioning a service domain interface, the SM API +# message expects the service domain name at the same offset +# as the service name field (used for provisioning a service) +SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD = 5 SM_API_MSG_PARAM = 6 @@ -98,3 +105,35 @@ def deprovision_service(service_name, service_group_name): SM_API_MSG_TYPE_DEPROVISION_SERVICE, "sm-action", service_name, service_group_name)) _send_msg_to_sm(sm_api_msg) + + +def provision_service_domain_interface(service_domain, + service_domain_interface_name): + """ + + :param service_domain: + :param service_domain_interface_name: + :return: + """ + sm_api_msg = ("%s,%s,%i,%s,%s,%s,%s" + % (SM_API_MSG_VERSION, SM_API_MSG_REVISION, 1, + SM_API_MSG_TYPE_PROVISION_SERVICE_DOMAIN_INTERFACE, + "sm-action", service_domain, + service_domain_interface_name)) + _send_msg_to_sm(sm_api_msg) + + +def deprovision_service_domain_interface(service_domain, + service_domain_interface_name): + """ + + :param service_domain: + :param service_domain_interface_name: + :return: + """ + sm_api_msg = ("%s,%s,%i,%s,%s,%s,%s" + % (SM_API_MSG_VERSION, SM_API_MSG_REVISION, 1, + SM_API_MSG_TYPE_DEPROVISION_SERVICE_DOMAIN_INTERFACE, + "sm-action", service_domain, + service_domain_interface_name)) + _send_msg_to_sm(sm_api_msg) diff --git a/service-mgmt-tools/sm-tools/sm_tools/sm_configure.py b/service-mgmt-tools/sm-tools/sm_tools/sm_configure.py index e491e40f..0ebf501e 100644 --- a/service-mgmt-tools/sm-tools/sm_tools/sm_configure.py +++ b/service-mgmt-tools/sm-tools/sm_tools/sm_configure.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2014-2015 Wind River Systems, Inc. +# Copyright (c) 2014-2023 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -15,9 +15,12 @@ cpe_duplex = "duplex" cpe_duplex_direct = "duplex-direct" mgmt_if = 'management-interface' cluster_host_if = 'cluster-host-interface' +admin_if = 'admin-interface' +admin_ip = 'admin-ip' tor_connect = 'tor' dc_connect = 'dc' database_name = "/var/lib/sm/sm.db" +runtime_db_name = "/var/run/sm/sm.db" def main(): @@ -41,6 +44,10 @@ def main(): if_parser.add_argument('network_peer_port', help='network peer port') if_parser.add_argument('network_peer_heartbeat_port', help='network peer heartbeat port') + if_parser.add_argument('--apply', + help='Apply the new configuration ' + 'immediately', + action='store_true') si_parser = subparsers.add_parser('service_instance', help='Service Instance ' @@ -49,6 +56,10 @@ def main(): si_parser.add_argument('service', help='service name') si_parser.add_argument('instance', help='instance name') si_parser.add_argument('parameters', help='instance parameters') + si_parser.add_argument('--apply', + help='Apply the new configuration ' + 'immediately', + action='store_true') sys_parser = subparsers.add_parser('system', help='system Configuration') @@ -110,6 +121,10 @@ def main(): print("Invalid sm_process_priority value. " "Must be between -2 to -20") sys.exit(-1) + elif args.which == 'interface': + _configure_interface(args) + elif args.which == 'service_instance': + _configure_service_instance(args) else: database = sqlite3.connect(database_name) _dispatch_config_action(args, database) @@ -233,6 +248,37 @@ def _dispatch_config_action(args, database): database.commit() +def _configure_interface(args): + database = sqlite3.connect(database_name) + _dispatch_config_action(args, database) + database.close() + + if args.apply: + if args.service_domain_interface == admin_if: + database = sqlite3.connect(runtime_db_name) + _dispatch_config_action(args, database) + database.close() + else: + print("Apply operation on service " + "domain interface %s is unsupported." % + (args.service_domain_interface)) + + +def _configure_service_instance(args): + database = sqlite3.connect(database_name) + _dispatch_config_action(args, database) + database.close() + if args.apply: + if args.service == admin_ip: + database = sqlite3.connect(runtime_db_name) + _dispatch_config_action(args, database) + database.close() + else: + print("Apply operation on service " + "instance %s is unsupported." % + (args.service)) + + def configure_cpe_duplex(): configure_if_connect_type(mgmt_if, tor_connect) configure_if_connect_type(cluster_host_if, tor_connect) diff --git a/service-mgmt-tools/sm-tools/sm_tools/sm_provision.py b/service-mgmt-tools/sm-tools/sm_tools/sm_provision.py index 0abb0ba7..034654f6 100755 --- a/service-mgmt-tools/sm-tools/sm_tools/sm_provision.py +++ b/service-mgmt-tools/sm-tools/sm_tools/sm_provision.py @@ -11,6 +11,8 @@ import argparse import sqlite3 from sm_tools.sm_api_msg_utils import provision_service from sm_tools.sm_api_msg_utils import deprovision_service +from sm_tools.sm_api_msg_utils import provision_service_domain_interface +from sm_tools.sm_api_msg_utils import deprovision_service_domain_interface database_name = "/var/lib/sm/sm.db" runtime_db_name = "/var/run/sm/sm.db" @@ -64,6 +66,10 @@ def main(): help='service domain name') sd_member_parser.add_argument('service_domain_interface', help='service domain interface name') + sd_member_parser.add_argument("--apply", + help="Apply the new configuration " + "immediately", + action="store_true") # Service-Group sg = subparsers.add_parser('service-group', @@ -120,18 +126,25 @@ def main(): database.close() elif args.which == 'service_domain_interface': - database = sqlite3.connect(database_name) + sql_update = "UPDATE SERVICE_DOMAIN_INTERFACES SET " \ + "PROVISIONED = '%s' WHERE SERVICE_DOMAIN = '%s' and " \ + "SERVICE_DOMAIN_INTERFACE = '%s';" \ + % (provision_str, args.service_domain, + args.service_domain_interface) + sqls = [sql_update] + update_db(database_name, sqls) - cursor = database.cursor() + if args.apply and os.path.isfile(runtime_db_name): + # update runtime configuration + update_db(runtime_db_name, sqls) - cursor.execute("UPDATE SERVICE_DOMAIN_INTERFACES SET " - "PROVISIONED = '%s' WHERE SERVICE_DOMAIN = '%s' and " - "SERVICE_DOMAIN_INTERFACE = '%s';" - % (provision_str, args.service_domain, - args.service_domain_interface)) - - database.commit() - database.close() + # tell SM to apply changes + if "yes" == provision_str: + provision_service_domain_interface(args.service_domain, + args.service_domain_interface) + elif "no" == provision_str: + deprovision_service_domain_interface(args.service_domain, + args.service_domain_interface) elif args.which == 'service_group': database = sqlite3.connect(database_name) diff --git a/service-mgmt/sm-common/src/sm_eru_process.c b/service-mgmt/sm-common/src/sm_eru_process.c index 69fdb2ed..1b78ec96 100644 --- a/service-mgmt/sm-common/src/sm_eru_process.c +++ b/service-mgmt/sm-common/src/sm_eru_process.c @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2017 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -115,6 +115,13 @@ void sm_eru_process_load_interfaces( void ) DPRINTFE( "Failed to look up cluster-host interface, error=%s.", sm_error_str(error) ); } + + error = sm_node_utils_get_admin_interface( &(_interfaces[3][0]) ); + if(( SM_OKAY != error )&&( SM_NOT_FOUND != error )) + { + DPRINTFE( "Failed to look up admin interface, error=%s.", + sm_error_str(error) ); + } } // **************************************************************************** diff --git a/service-mgmt/sm-common/src/sm_node_utils.c b/service-mgmt/sm-common/src/sm_node_utils.c index 2eb9f6d7..ade940c9 100644 --- a/service-mgmt/sm-common/src/sm_node_utils.c +++ b/service-mgmt/sm-common/src/sm_node_utils.c @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2020 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -222,6 +222,17 @@ SmErrorT sm_node_utils_get_cluster_host_interface( char interface_name[] ) } // **************************************************************************** +// **************************************************************************** +// Node Utilities - Get Admin Interface +// =========================================== +SmErrorT sm_node_utils_get_admin_interface( char interface_name[] ) +{ + return( sm_node_utils_read_platform_config( "admin_interface", + interface_name, + SM_INTERFACE_NAME_MAX_CHAR ) ); +} +// **************************************************************************** + // **************************************************************************** // Node Utilities - Get system mode string // ============================================= diff --git a/service-mgmt/sm-common/src/sm_node_utils.h b/service-mgmt/sm-common/src/sm_node_utils.h index fccae49a..2257a4a4 100644 --- a/service-mgmt/sm-common/src/sm_node_utils.h +++ b/service-mgmt/sm-common/src/sm_node_utils.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2014,2020 Wind River Systems, Inc. +// Copyright (c) 2014,2020-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -50,6 +50,12 @@ extern SmErrorT sm_node_utils_get_oam_interface( char interface_name[] ); extern SmErrorT sm_node_utils_get_cluster_host_interface( char interface_name[] ); // **************************************************************************** +// **************************************************************************** +// Node Utilities - Get Admin Interface +// =========================================== +extern SmErrorT sm_node_utils_get_admin_interface( char interface_name[] ); +// **************************************************************************** + // **************************************************************************** // Node Utilities - Get system mode // ============================================= diff --git a/service-mgmt/sm-common/src/sm_types.c b/service-mgmt/sm-common/src/sm_types.c index 09e5e116..eb9d4a4b 100644 --- a/service-mgmt/sm-common/src/sm_types.c +++ b/service-mgmt/sm-common/src/sm_types.c @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2020 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -1872,6 +1872,9 @@ SmInterfaceTypeT sm_get_interface_type( const char* domain_interface ) } else if ( 0 == strcmp ( SM_SERVICE_DOMAIN_CLUSTER_HOST_INTERFACE, domain_interface ) ) { return SM_INTERFACE_CLUSTER_HOST; + } else if ( 0 == strcmp ( SM_SERVICE_DOMAIN_ADMIN_INTERFACE, domain_interface ) ) + { + return SM_INTERFACE_ADMIN; } return SM_INTERFACE_UNKNOWN; diff --git a/service-mgmt/sm-common/src/sm_types.h b/service-mgmt/sm-common/src/sm_types.h index ec843e15..25717a9f 100644 --- a/service-mgmt/sm-common/src/sm_types.h +++ b/service-mgmt/sm-common/src/sm_types.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2020 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -96,10 +96,12 @@ extern "C" { #define SM_SERVICE_DOMAIN_MGMT_INTERFACE "management-interface" #define SM_SERVICE_DOMAIN_OAM_INTERFACE "oam-interface" #define SM_SERVICE_DOMAIN_CLUSTER_HOST_INTERFACE "cluster-host-interface" +#define SM_SERVICE_DOMAIN_ADMIN_INTERFACE "admin-interface" #define SM_MGMT_INTERFACE_NAME "mgmt" #define SM_OAM_INTERFACE_NAME "oam" #define SM_CLUSTER_HOST_INTERFACE_NAME "cluster-host" +#define SM_ADMIN_INTERFACE_NAME "admin" #define SM_MAX_IF_NAME_LEN 5 #define SM_NODE_CONTROLLER_0_NAME "controller-0" @@ -201,7 +203,8 @@ typedef enum SM_INTERFACE_UNKNOWN, SM_INTERFACE_MGMT, SM_INTERFACE_CLUSTER_HOST, - SM_INTERFACE_OAM + SM_INTERFACE_OAM, + SM_INTERFACE_ADMIN }SmInterfaceTypeT; typedef enum diff --git a/service-mgmt/sm-db/database/create_sm_db.sql b/service-mgmt/sm-db/database/create_sm_db.sql index 4e015896..3b6feb61 100644 --- a/service-mgmt/sm-db/database/create_sm_db.sql +++ b/service-mgmt/sm-db/database/create_sm_db.sql @@ -6,6 +6,7 @@ CREATE TABLE SERVICE_DOMAIN_INTERFACES ( ID INTEGER PRIMARY KEY AUTOINCREMENT, P INSERT INTO "SERVICE_DOMAIN_INTERFACES" VALUES(1,'yes','controller','management-interface','secondary','none','','','','','','','','','','','','tor'); INSERT INTO "SERVICE_DOMAIN_INTERFACES" VALUES(2,'yes','controller','oam-interface','secondary','hmac-sha512','titanium-server','','','','','','','','','','','tor'); INSERT INTO "SERVICE_DOMAIN_INTERFACES" VALUES(3,'yes','controller','cluster-host-interface','secondary','none','','','','','','','','','','','','tor'); +INSERT INTO "SERVICE_DOMAIN_INTERFACES" VALUES(4,'no','controller','admin-interface','secondary','none','','','','','','','','','','','','tor'); CREATE TABLE SERVICE_DOMAINS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), ORCHESTRATION CHAR(32), DESIGNATION CHAR(32), PREEMPT CHAR(32), PRIORITY INT, HELLO_INTERVAL INT, DEAD_INTERVAL INT, WAIT_INTERVAL INT, EXCHANGE_INTERVAL INT, STATE CHAR(32), SPLIT_BRAIN_RECOVERY CHAR(32), LEADER CHAR(32), GENERATION INT); INSERT INTO "SERVICE_DOMAINS" VALUES(1,'yes','controller','regional','unknown','no',230,200,800,5000,2000,'initial','select-best-active','',1); CREATE TABLE SERVICE_DOMAIN_MEMBERS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), SERVICE_GROUP_NAME CHAR(32), REDUNDANCY_MODEL CHAR(32), N_ACTIVE INT, M_STANDBY INT, SERVICE_GROUP_AGGREGATE CHAR(32), ACTIVE_ONLY_IF_ACTIVE CHAR(32) ); @@ -19,6 +20,7 @@ INSERT INTO "SERVICE_DOMAIN_MEMBERS" VALUES(7,'yes','controller','web-services', INSERT INTO "SERVICE_DOMAIN_MEMBERS" VALUES(8,'no','controller','storage-services','N',2,0,'',''); INSERT INTO "SERVICE_DOMAIN_MEMBERS" VALUES(9,'no','controller','storage-monitoring-services','N + M',1,1,'',''); INSERT INTO "SERVICE_DOMAIN_MEMBERS" VALUES(10,'no','controller','distributed-cloud-services','N + M',1,1,'controller-aggregate',''); +INSERT INTO "SERVICE_DOMAIN_MEMBERS" VALUES(11,'no','controller','admin-services','N + M',1,1,'controller-aggregate','directory-services'); CREATE TABLE SERVICE_DOMAIN_NEIGHBORS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, NAME CHAR(32), SERVICE_DOMAIN CHAR(32), ORCHESTRATION CHAR(32), DESIGNATION CHAR(32), PRIORITY INT, HELLO_INTERVAL INT, DEAD_INTERVAL INT, WAIT_INTERVAL INT, EXCHANGE_INTERVAL INT, STATE CHAR(32), GENERATION INT); CREATE TABLE SERVICE_DOMAIN_ASSIGNMENTS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, UUID CHAR(42), NAME CHAR(32), NODE_NAME CHAR(32), SERVICE_GROUP_NAME CHAR(32), DESIRED_STATE CHAR(32), STATE CHAR(32), STATUS CHAR(32), CONDITION CHAR(32) ); CREATE TABLE SERVICE_GROUPS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), AUTO_RECOVER CHAR(32), CORE CHAR(32), DESIRED_STATE CHAR(32), STATE CHAR(32), STATUS CHAR(32), CONDITION CHAR(32), FAILURE_DEBOUNCE INT, FATAL_ERROR_REBOOT CHAR(32) ); @@ -32,6 +34,7 @@ INSERT INTO "SERVICE_GROUPS" VALUES(7,'no','storage-services','no','no','initial INSERT INTO "SERVICE_GROUPS" VALUES(8,'no','storage-monitoring-services','no','no','initial','initial','','',120000,'no'); INSERT INTO "SERVICE_GROUPS" VALUES(9,'yes','vim-services','no','no','initial','initial','','',300000,'yes'); INSERT INTO "SERVICE_GROUPS" VALUES(10,'no','distributed-cloud-services','no','no','initial','initial','','',300000,'yes'); +INSERT INTO "SERVICE_GROUPS" VALUES(11,'no','admin-services','no','no','initial','initial','','',300000,'no'); CREATE TABLE SERVICE_GROUP_MEMBERS ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), SERVICE_NAME CHAR(32), SERVICE_FAILURE_IMPACT CHAR(32)); INSERT INTO "SERVICE_GROUP_MEMBERS" VALUES(1,'yes','oam-services','oam-ip','critical'); INSERT INTO "SERVICE_GROUP_MEMBERS" VALUES(2,'yes','controller-services','management-ip','critical'); @@ -95,6 +98,7 @@ INSERT INTO "SERVICE_GROUP_MEMBERS" SELECT MAX(id) + 1,'no','distributed-cloud-s INSERT INTO "SERVICE_GROUP_MEMBERS" SELECT MAX(id) + 1,'no','distributed-cloud-services','dcmanager-orchestrator','critical' FROM "SERVICE_GROUP_MEMBERS"; INSERT INTO "SERVICE_GROUP_MEMBERS" SELECT MAX(id) + 1,'no','distributed-cloud-services','dcmanager-audit-worker','critical' FROM "SERVICE_GROUP_MEMBERS"; INSERT INTO "SERVICE_GROUP_MEMBERS" SELECT MAX(id) + 1,'no','distributed-cloud-services','dcmanager-state','major' FROM "SERVICE_GROUP_MEMBERS"; +INSERT INTO "SERVICE_GROUP_MEMBERS" SELECT MAX(id) + 1,'no','admin-services','admin-ip','critical' FROM "SERVICE_GROUP_MEMBERS"; CREATE TABLE SERVICES ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), DESIRED_STATE CHAR(32), STATE CHAR(32), STATUS CHAR(32), CONDITION CHAR(32), MAX_FAILURES INT, FAIL_COUNTDOWN INT, FAIL_COUNTDOWN_INTERVAL INT, MAX_ACTION_FAILURES INT, MAX_TRANSITION_FAILURES INT, PID_FILE CHAR(256) ); INSERT INTO "SERVICES" VALUES(1,'yes','oam-ip','initial','initial','none','none',2,1,90000,4,16,''); INSERT INTO "SERVICES" VALUES(2,'yes','management-ip','initial','initial','none','none',2,1,90000,4,16,''); @@ -158,6 +162,7 @@ INSERT INTO "SERVICES" SELECT MAX(id) + 1,'no','dcdbsync-openstack-api','initial INSERT INTO "SERVICES" SELECT MAX(id) + 1,'no','dcmanager-orchestrator','initial','initial','none','none',2,1,90000,4,16,'/var/run/resource-agents/dcmanager-orchestrator.pid' FROM "SERVICES"; INSERT INTO "SERVICES" SELECT MAX(id) + 1,'no','dcmanager-audit-worker','initial','initial','none','none',2,1,90000,4,16,'/var/run/resource-agents/dcmanager-audit-worker.pid' FROM "SERVICES"; INSERT INTO "SERVICES" SELECT MAX(id) + 1,'no','dcmanager-state','initial','initial','none','none',2,1,90000,4,16,'/var/run/resource-agents/dcmanager-state.pid' FROM "SERVICES"; +INSERT INTO "SERVICES" SELECT MAX(id) + 1,'no','admin-ip','initial','initial','none','none',2,1,90000,4,16,'' FROM "SERVICES"; CREATE TABLE SERVICE_HEARTBEAT ( ID INTEGER PRIMARY KEY AUTOINCREMENT, PROVISIONED CHAR(32), NAME CHAR(32), TYPE CHAR(32), SRC_ADDRESS CHAR(108), SRC_PORT INT, DST_ADDRESS CHAR(108), DST_PORT INT, MESSAGE CHAR(256), INTERVAL_IN_MS INT, MISSED_WARN INT, MISSED_DEGRADE INT, MISSED_FAIL INT, STATE CHAR(32), MISSED INT, HEARTBEAT_TIMER_ID INT, HEARTBEAT_SOCKET INT ); CREATE TABLE SERVICE_DEPENDENCY ( DEPENDENCY_TYPE CHAR(32), SERVICE_NAME CHAR(32), STATE CHAR(32), ACTION CHAR(32), DEPENDENT CHAR(32), DEPENDENT_STATE CHAR(32), PRIMARY KEY (DEPENDENCY_TYPE, SERVICE_NAME, STATE, ACTION, DEPENDENT)); INSERT INTO "SERVICE_DEPENDENCY" VALUES('action','oam-ip','not-applicable','enable','management-ip','enabled-active'); @@ -576,6 +581,10 @@ INSERT INTO "SERVICE_ACTIONS" VALUES('dcmanager-state','enable','ocf-script','op INSERT INTO "SERVICE_ACTIONS" VALUES('dcmanager-state','disable','ocf-script','openstack','dcmanager-state','stop','',1,1,1,20,''); INSERT INTO "SERVICE_ACTIONS" VALUES('dcmanager-state','audit-enabled','ocf-script','openstack','dcmanager-state','monitor','',2,2,2,20,5); INSERT INTO "SERVICE_ACTIONS" VALUES('dcmanager-state','audit-disabled','ocf-script','openstack','dcmanager-state','monitor','',0,0,0,20,5); +INSERT INTO "SERVICE_ACTIONS" VALUES('admin-ip','enable','ocf-script','heartbeat','IPaddr2','start','',2,2,2,20,''); +INSERT INTO "SERVICE_ACTIONS" VALUES('admin-ip','disable','ocf-script','heartbeat','IPaddr2','stop','',1,1,1,20,''); +INSERT INTO "SERVICE_ACTIONS" VALUES('admin-ip','audit-enabled','ocf-script','heartbeat','IPaddr2','monitor','',2,2,2,20,5); +INSERT INTO "SERVICE_ACTIONS" VALUES('admin-ip','audit-disabled','ocf-script','heartbeat','IPaddr2','monitor','',0,0,0,20,5); CREATE TABLE SERVICE_ACTION_RESULTS ( PLUGIN_TYPE CHAR(32), PLUGIN_NAME CHAR(80), PLUGIN_COMMAND CHAR(80), PLUGIN_EXIT_CODE CHAR(32), ACTION_RESULT CHAR(32), SERVICE_STATE CHAR(32), SERVICE_STATUS CHAR(32), SERVICE_CONDITION CHAR(32), PRIMARY KEY (PLUGIN_TYPE, PLUGIN_NAME, PLUGIN_COMMAND, PLUGIN_EXIT_CODE)); INSERT INTO "SERVICE_ACTION_RESULTS" VALUES('lsb-script','default','status','0','success','enabled-active','unknown','unknown'); INSERT INTO "SERVICE_ACTION_RESULTS" VALUES('lsb-script','default','status','1','success','disabled','unknown','unknown'); diff --git a/service-mgmt/sm/src/sm_api.c b/service-mgmt/sm/src/sm_api.c index 55b53af5..989bf434 100644 --- a/service-mgmt/sm/src/sm_api.c +++ b/service-mgmt/sm/src/sm_api.c @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2019 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -32,7 +32,8 @@ #define SM_API_MSG_SKIP_DEP_CHECK "skip-dep" #define SM_API_MSG_TYPE_PROVISION_SERVICE "PROVISION_SERVICE" #define SM_API_MSG_TYPE_DEPROVISION_SERVICE "DEPROVISION_SERVICE" - +#define SM_API_MSG_TYPE_PROVISION_SERVICE_DOMAIN_INTERFACE "PROVISION_SERVICE_DOMAIN_INTERFACE" +#define SM_API_MSG_TYPE_DEPROVISION_SERVICE_DOMAIN_INTERFACE "DEPROVISION_SERVICE_DOMAIN_INTERFACE" #define SM_API_MSG_NODE_ACTION_UNKNOWN "unknown" #define SM_API_MSG_NODE_ACTION_LOCK "lock" @@ -67,6 +68,9 @@ #define SM_API_MSG_NODE_AVAIL_FIELD 9 #define SM_API_MSG_SERVICE_NAME_FIELD 5 +// The service domain interface provision / deprovision msg is at the same +// message offset as the service name field. +#define SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD 5 #define SM_API_MSG_PARAM 6 #define SM_API_MAX_MSG_SIZE 2048 @@ -379,6 +383,8 @@ static void sm_api_dispatch( int selobj, int64_t user_data ) char* node_name; char* service_name; char* service_group_name; + char* service_domain; + char* service_domain_interface; gchar** params; int bytes_read; SmNodeSetActionT action; @@ -445,7 +451,7 @@ static void sm_api_dispatch( int selobj, int64_t user_data ) DPRINTFE( "Missing message-type field in received message." ); goto ERROR; } - + if( 0 == strcmp( SM_API_MSG_TYPE_SET_NODE, params[SM_API_MSG_TYPE_FIELD] ) ) { if( params[SM_API_MSG_ORIGIN_FIELD] == NULL ) @@ -598,6 +604,72 @@ static void sm_api_dispatch( int selobj, int64_t user_data ) { _callbacks.deprovision_service( service_group_name, service_name, seqno); } + } + else if( 0 == strcmp( SM_API_MSG_TYPE_PROVISION_SERVICE_DOMAIN_INTERFACE, + params[SM_API_MSG_TYPE_FIELD] ) ) + { + if( params[SM_API_MSG_ORIGIN_FIELD] == NULL ) + { + DPRINTFE( "Missing origin field in received message." ); + goto ERROR; + } + + if( params[SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD] == NULL ) + { + DPRINTFE( "Missing service-domain field in received message." ); + goto ERROR; + } + service_domain = (char*) params[SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD]; + + service_domain_interface = (char*) params[SM_API_MSG_PARAM]; + if ( NULL == service_domain_interface ) + { + DPRINTFE( "Missing service domain interface parameter." ); + goto ERROR; + } + else + { + DPRINTFI( "Provision service domain interface %s:%s", + service_domain, service_domain_interface); + } + + if( NULL != _callbacks.provision_service_domain_interface ) + { + _callbacks.provision_service_domain_interface( service_domain, service_domain_interface, seqno); + } + } + else if( 0 == strcmp( SM_API_MSG_TYPE_DEPROVISION_SERVICE_DOMAIN_INTERFACE, + params[SM_API_MSG_TYPE_FIELD] ) ) + { + if( params[SM_API_MSG_ORIGIN_FIELD] == NULL ) + { + DPRINTFE( "Missing origin field in received message." ); + goto ERROR; + } + + if( params[SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD] == NULL ) + { + DPRINTFE( "Missing service-domain field in received message." ); + goto ERROR; + } + service_domain = (char*) params[SM_API_MSG_SERVICE_DOMAIN_NAME_FIELD]; + + service_domain_interface = (char*) params[SM_API_MSG_PARAM]; + if ( NULL == service_domain_interface ) + { + DPRINTFE( "Missing service domain interface parameter." ); + goto ERROR; + } + else + { + DPRINTFI( "Deprovision service domain interface %s:%s", + service_domain, service_domain_interface); + } + + if( NULL != _callbacks.deprovision_service_domain_interface ) + { + _callbacks.deprovision_service_domain_interface( service_domain, service_domain_interface, seqno); + } } else { DPRINTFE( "Unknown/unsupported message-type (%s) received.", params[SM_API_MSG_TYPE_FIELD] ); diff --git a/service-mgmt/sm/src/sm_api.h b/service-mgmt/sm/src/sm_api.h index 52db0134..fa50d85d 100644 --- a/service-mgmt/sm/src/sm_api.h +++ b/service-mgmt/sm/src/sm_api.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2019 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -37,12 +37,23 @@ typedef void (*SmApiProvisionServiceCallbackT) (char service_group_name[], typedef void (*SmApiDeprovisionServiceCallbackT) (char service_group_name[], char service_name[], int seqno); +typedef void (*SmApiProvisionServiceDomainInterfaceCallbackT) (char service_domain[], + char interface_name[], int seqno); + +typedef void (*SmApiDeprovisionServiceDomainInterfaceCallbackT) (char service_domain[], + char interface_name[], int seqno); + +typedef void (*SmApiConfigureServiceDomainInterfaceCallbackT) (char service_domain[], + char service_domain_interface[], int seqno); + typedef struct { SmApiNodeSetCallbackT node_set; SmApiServiceRestartCallbackT service_restart; SmApiProvisionServiceCallbackT provision_service; SmApiDeprovisionServiceCallbackT deprovision_service; + SmApiProvisionServiceDomainInterfaceCallbackT provision_service_domain_interface; + SmApiDeprovisionServiceDomainInterfaceCallbackT deprovision_service_domain_interface; } SmApiCallbacksT; // **************************************************************************** diff --git a/service-mgmt/sm/src/sm_configure.cpp b/service-mgmt/sm/src/sm_configure.cpp index e3b1d277..fb6f1cfa 100644 --- a/service-mgmt/sm/src/sm_configure.cpp +++ b/service-mgmt/sm/src/sm_configure.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2019 Wind River Systems, Inc. +// Copyright (c) 2019-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -20,6 +20,10 @@ #include "sm_service_domain_assignment_table.h" #include "sm_timer.h" #include "sm_service_fsm.h" +#include "sm_service_domain_interface_api.h" +#include "sm_service_domain_interface_table.h" +#include "sm_heartbeat.h" +#include "sm_msg.h" // The behavior of a service is configured in multiple (>1) service groups has @@ -184,3 +188,82 @@ SmErrorT sm_deprovision_service( char service_group_name[], char service_name[] return SM_OKAY; } // **************************************************************************** + +// **************************************************************************** +// sm configure: provision service domain interface +// ================================ +extern SmErrorT sm_provision_service_domain_interface( char service_domain_name[], char interface_name[] ) +{ + SmErrorT error; + + DPRINTFI("Start provisioning service domain interface %s:%s...", service_domain_name, interface_name); + + error = sm_service_domain_interface_table_load(); + if(SM_OKAY != error) + { + DPRINTFE("Failed to reload domain interface table"); + return error; + } + + SmServiceDomainInterfaceT* interface = sm_service_domain_interface_table_read(service_domain_name, interface_name); + if(NULL == interface) + { + DPRINTFI("Service domain interface %s not found", interface_name); + return SM_NOT_FOUND; + } + + error = sm_service_domain_interface_api_provisioned(interface); + if(SM_OKAY != error) + { + DPRINTFE("Failed to provision service domain interface %s:%s", service_domain_name, interface_name); + return error; + } + + error = sm_service_domain_interface_table_load(); + if(SM_OKAY != error) + { + DPRINTFE("Failed to reload domain interface table"); + return error; + } + + error = sm_service_domain_interface_api_audit(); + if(SM_OKAY != error) + { + DPRINTFE( "Failed to audit interfaces, error=%s.", + sm_error_str( error ) ); + } + + DPRINTFI("%s:%s is provisioned successfully", service_domain_name, interface_name); + return SM_OKAY; + +} +// **************************************************************************** + +// **************************************************************************** +// sm configure: deprovision service domain interface +// =============================== +SmErrorT sm_deprovision_service_domain_interface( char service_domain_name[], char interface_name[] ) +{ + SmErrorT error; + DPRINTFI("Start deprovisioning service domain interface %s:%s ...", service_domain_name, interface_name); + + SmServiceDomainInterfaceT* interface = sm_service_domain_interface_table_read(service_domain_name, interface_name); + if(NULL == interface) + { + DPRINTFI("Service domain interface %s not found", interface_name); + return SM_NOT_FOUND; + } + + error = sm_service_domain_interface_api_deprovisioned(interface); + if(SM_OKAY != error) + { + DPRINTFE("Failed to deprovision service domain interface %s:%s", service_domain_name, interface_name); + return error; + } + + DPRINTFI("%s:%s is deprovisioned successfully", service_domain_name, interface_name); + + return SM_OKAY; +} +// **************************************************************************** + diff --git a/service-mgmt/sm/src/sm_configure.h b/service-mgmt/sm/src/sm_configure.h index 9917908b..524eedb2 100644 --- a/service-mgmt/sm/src/sm_configure.h +++ b/service-mgmt/sm/src/sm_configure.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2019 Wind River Systems, Inc. +// Copyright (c) 2019-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -24,6 +24,18 @@ extern SmErrorT sm_provision_service( char service_group_name[], char service_na extern SmErrorT sm_deprovision_service( char service_group_name[], char service_name[] ); // **************************************************************************** +// **************************************************************************** +// system config: provision_service_domain_interface +// ================================ +extern SmErrorT sm_provision_service_domain_interface( char service_domain_name[], char interface_name[] ); +// **************************************************************************** + +// **************************************************************************** +// system config: deprovision_service_domain_interface +// ================================ +extern SmErrorT sm_deprovision_service_domain_interface( char service_domain_name[], char interface_name[] ); +// **************************************************************************** + #ifdef __cplusplus } diff --git a/service-mgmt/sm/src/sm_failover.c b/service-mgmt/sm/src/sm_failover.c index a250649b..9ea5970c 100644 --- a/service-mgmt/sm/src/sm_failover.c +++ b/service-mgmt/sm/src/sm_failover.c @@ -1,5 +1,5 @@ // -// Copyright (c) 2017 Wind River Systems, Inc. +// Copyright (c) 2017-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -137,6 +137,7 @@ static unsigned int _total_interfaces; static SmFailoverInterfaceInfo* _oam_interface_info = NULL; static SmFailoverInterfaceInfo* _mgmt_interface_info = NULL; static SmFailoverInterfaceInfo* _cluster_host_interface_info = NULL; +static SmFailoverInterfaceInfo* _admin_interface_info = NULL; static SmFailoverInterfaceInfo _peer_if_list[SM_INTERFACE_MAX]; static pthread_mutex_t _mutex; static SmDbHandleT* _sm_db_handle = NULL; @@ -193,6 +194,19 @@ static void sm_failover_interface_check( void* user_data[], DPRINTFE( "Failed to look up cluster-host interface, error=%s.", sm_error_str(error) ); } + }else if( 0 == strcmp(SM_SERVICE_DOMAIN_ADMIN_INTERFACE, interface->service_domain_interface )) + { + SmErrorT error = sm_node_utils_get_admin_interface(interface->interface_name); + if(SM_OKAY == error) + { + _my_if_list[*count].set_interface(interface); + _admin_interface_info = _my_if_list + (*count); + (*count) ++; + } else if (SM_NOT_FOUND != error ) + { + DPRINTFE( "Failed to look up admin interface, error=%s.", + sm_error_str(error) ); + } }else { DPRINTFE( "Unknown interface %s", interface->interface_name ); @@ -483,6 +497,21 @@ bool is_cluster_host_interface_configured() } // **************************************************************************** +// **************************************************************************** +// Failover - is admin interface configured +// =============================================== +bool is_admin_interface_configured() +{ + if (NULL == _admin_interface_info || + _admin_interface_info->get_interface()->service_domain_interface[0] == '\0' ) + { + return false; + } + + return true; +} +// **************************************************************************** + // **************************************************************************** // Failover - get interface state // ================== @@ -491,6 +520,7 @@ int sm_failover_get_if_state() SmFailoverInterfaceStateT mgmt_state = _mgmt_interface_info->get_state(); SmFailoverInterfaceStateT oam_state = _oam_interface_info->get_state(); SmFailoverInterfaceStateT cluster_host_state; + SmFailoverInterfaceStateT admin_state; int if_state_flag = 0; if ( is_cluster_host_interface_configured() ) { @@ -505,6 +535,19 @@ int sm_failover_get_if_state() } } + if ( is_admin_interface_configured() ) + { + admin_state = _admin_interface_info->get_state(); + if( SM_FAILOVER_INTERFACE_OK == admin_state ) + { + if_state_flag |= SM_FAILOVER_HEARTBEAT_ALIVE; + } + else if ( SM_FAILOVER_INTERFACE_DOWN == admin_state ) + { + if_state_flag |= SM_FAILOVER_ADMIN_DOWN; + } + } + if( SM_FAILOVER_INTERFACE_OK == mgmt_state ) { if_state_flag |= SM_FAILOVER_HEARTBEAT_ALIVE; @@ -1018,6 +1061,13 @@ void sm_failover_audit() { event_data.set_interface_state(SM_INTERFACE_CLUSTER_HOST, SM_FAILOVER_INTERFACE_UNKNOWN); } + if( NULL != _admin_interface_info) + { + event_data.set_interface_state(SM_INTERFACE_ADMIN, _admin_interface_info->get_state()); + }else + { + event_data.set_interface_state(SM_INTERFACE_ADMIN, SM_FAILOVER_INTERFACE_UNKNOWN); + } SmFailoverFSM::get_fsm().send_event(SM_FAILOVER_EVENT_IF_STATE_CHANGED, &event_data); _if_state_changed = false; } @@ -1186,6 +1236,9 @@ SmFailoverInterfaceStateT sm_failover_get_interface_info(SmInterfaceTypeT interf case SM_INTERFACE_OAM: res = _oam_interface_info; break; + case SM_INTERFACE_ADMIN: + res = _admin_interface_info; + break; case SM_INTERFACE_UNKNOWN: break; } @@ -1471,6 +1524,7 @@ void dump_interfaces_state(FILE* fp) dump_if_state(fp, _oam_interface_info, " OAM"); dump_if_state(fp, _mgmt_interface_info, " MGMT"); dump_if_state(fp, _cluster_host_interface_info, "CLUSTER-HOST"); + dump_if_state(fp, _cluster_host_interface_info, "ADMIN"); } void dump_peer_if_state(FILE* fp) diff --git a/service-mgmt/sm/src/sm_failover.h b/service-mgmt/sm/src/sm_failover.h index 1c6dd6e1..82823dd6 100644 --- a/service-mgmt/sm/src/sm_failover.h +++ b/service-mgmt/sm/src/sm_failover.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2017-2020 Wind River Systems, Inc. +// Copyright (c) 2017-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -36,6 +36,7 @@ typedef enum SM_FAILOVER_OAM_DOWN = 4, SM_FAILOVER_HEARTBEAT_ALIVE = 8, SM_FAILOVER_HELLO_MSG_ALIVE = 16, + SM_FAILOVER_ADMIN_DOWN = 32, SM_FAILOVER_PEER_DISABLED = 0x4000, }SmFailoverCommFaultBitFlagT; diff --git a/service-mgmt/sm/src/sm_failover_fail_pending_state.cpp b/service-mgmt/sm/src/sm_failover_fail_pending_state.cpp index 45cf89f0..c80c10e6 100644 --- a/service-mgmt/sm/src/sm_failover_fail_pending_state.cpp +++ b/service-mgmt/sm/src/sm_failover_fail_pending_state.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2018 Wind River Systems, Inc. +// Copyright (c) 2018-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -235,6 +235,7 @@ SmErrorT SmFailoverFailPendingState::event_handler(SmFailoverEventT event, const { interfaces_to_check[0] = SM_INTERFACE_MGMT; interfaces_to_check[1] = SM_INTERFACE_CLUSTER_HOST; + interfaces_to_check[2] = SM_INTERFACE_ADMIN; } for(int i = 0; interfaces_to_check[i] != SM_INTERFACE_UNKNOWN; i ++) { diff --git a/service-mgmt/sm/src/sm_failover_failed_state.cpp b/service-mgmt/sm/src/sm_failover_failed_state.cpp index 59aaf17f..59821dd5 100644 --- a/service-mgmt/sm/src/sm_failover_failed_state.cpp +++ b/service-mgmt/sm/src/sm_failover_failed_state.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2018-2021 Wind River Systems, Inc. +// Copyright (c) 2018-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -28,6 +28,7 @@ #include "sm_cluster_hbs_info_msg.h" extern bool is_cluster_host_interface_configured( void ); +extern bool is_admin_interface_configured( void ); // Failover Failed Recovery Audit period = 5 seconds static const int FAILED_STATE_AUDIT_PERIOD = 5000; @@ -147,9 +148,10 @@ static bool sm_failover_failed_recovery_criteria_met( void ) { bool criteria_met = false ; - SmFailoverInterfaceStateT oam_state, mgmt_state, cluster_host_state; + SmFailoverInterfaceStateT oam_state, mgmt_state, cluster_host_state, admin_state; oam_state = sm_failover_get_interface_info(SM_INTERFACE_OAM); mgmt_state = sm_failover_get_interface_info(SM_INTERFACE_MGMT); + admin_state = sm_failover_get_interface_info(SM_INTERFACE_ADMIN); const SmClusterHbsStateT& cluster_hbs_state = SmClusterHbsInfoMsg::get_current_state(); int peer_controller_index = SmClusterHbsInfoMsg::get_peer_controller_index(); @@ -172,12 +174,23 @@ static bool sm_failover_failed_recovery_criteria_met( void ) { criteria_met = true ; } + + if ( criteria_met && is_admin_interface_configured() ) + { + criteria_met = false ; + admin_state = sm_failover_get_interface_info(SM_INTERFACE_ADMIN); + if (( admin_state == SM_FAILOVER_INTERFACE_OK ) || ( admin_state == SM_FAILOVER_INTERFACE_MISSING_HEARTBEAT )) + { + criteria_met = true; + } + } } - DPRINTFI("Oam:%s ; Mgmt:%s ; Cluster:%s ; recovery criteria met: %s", + DPRINTFI("Oam:%s ; Mgmt:%s ; Cluster:%s ; Admin:%s recovery criteria met: %s", sm_failover_interface_state_str(oam_state), sm_failover_interface_state_str(mgmt_state), sm_failover_interface_state_str(cluster_host_state), + sm_failover_interface_state_str(admin_state), criteria_met ? "Yes" : "No"); return (criteria_met); diff --git a/service-mgmt/sm/src/sm_failover_fsm.cpp b/service-mgmt/sm/src/sm_failover_fsm.cpp index 7b5c94c1..07b815a8 100644 --- a/service-mgmt/sm/src/sm_failover_fsm.cpp +++ b/service-mgmt/sm/src/sm_failover_fsm.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2018 Wind River Systems, Inc. +// Copyright (c) 2018-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -36,6 +36,9 @@ void SmIFStateChangedEventData::set_interface_state( case SM_INTERFACE_CLUSTER_HOST: _cluster_host_state = interface_state; break; + case SM_INTERFACE_ADMIN: + _admin_state = interface_state; + break; default: DPRINTFE("Runtime error: invalid interface type %d", interface_type); } @@ -51,6 +54,8 @@ SmFailoverInterfaceStateT SmIFStateChangedEventData::get_interface_state(SmInter return _mgmt_state; case SM_INTERFACE_CLUSTER_HOST: return _cluster_host_state; + case SM_INTERFACE_ADMIN: + return _admin_state; default: DPRINTFE("Runtime error: invalid interface type %d", interface_type); return SM_FAILOVER_INTERFACE_UNKNOWN; diff --git a/service-mgmt/sm/src/sm_failover_fsm.h b/service-mgmt/sm/src/sm_failover_fsm.h index 5614b54f..391b3a2e 100644 --- a/service-mgmt/sm/src/sm_failover_fsm.h +++ b/service-mgmt/sm/src/sm_failover_fsm.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2018 Wind River Systems, Inc. +// Copyright (c) 2018-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -73,6 +73,7 @@ class SmIFStateChangedEventData: public ISmFSMEventData SmFailoverInterfaceStateT _mgmt_state; SmFailoverInterfaceStateT _cluster_host_state; SmFailoverInterfaceStateT _oam_state; + SmFailoverInterfaceStateT _admin_state; }; #endif //__SM_FAILOVER_FSM_H__ diff --git a/service-mgmt/sm/src/sm_failover_normal_state.cpp b/service-mgmt/sm/src/sm_failover_normal_state.cpp index 1546fe73..457a0496 100644 --- a/service-mgmt/sm/src/sm_failover_normal_state.cpp +++ b/service-mgmt/sm/src/sm_failover_normal_state.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2018 Wind River Systems, Inc. +// Copyright (c) 2018-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -25,13 +25,15 @@ SmErrorT SmFailoverNormalState::event_handler(SmFailoverEventT event, const ISmF { const SmIFStateChangedEventData* data = (const SmIFStateChangedEventData*) event_data; - SmFailoverInterfaceStateT oam_state, mgmt_state, cluster_host_state; + SmFailoverInterfaceStateT oam_state, mgmt_state, cluster_host_state, admin_state; oam_state = data->get_interface_state(SM_INTERFACE_OAM); mgmt_state = data->get_interface_state(SM_INTERFACE_MGMT); cluster_host_state = data->get_interface_state(SM_INTERFACE_CLUSTER_HOST); + admin_state = data->get_interface_state(SM_INTERFACE_ADMIN); if(oam_state != SM_FAILOVER_INTERFACE_OK || mgmt_state != SM_FAILOVER_INTERFACE_OK || - (cluster_host_state != SM_FAILOVER_INTERFACE_OK && cluster_host_state != SM_FAILOVER_INTERFACE_UNKNOWN)) + (cluster_host_state != SM_FAILOVER_INTERFACE_OK && cluster_host_state != SM_FAILOVER_INTERFACE_UNKNOWN) || + (admin_state != SM_FAILOVER_INTERFACE_OK && admin_state != SM_FAILOVER_INTERFACE_UNKNOWN)) { this->fsm.set_state(SM_FAILOVER_STATE_FAIL_PENDING); } diff --git a/service-mgmt/sm/src/sm_failover_ss.c b/service-mgmt/sm/src/sm_failover_ss.c index 32a5f76c..1ac7eb69 100644 --- a/service-mgmt/sm/src/sm_failover_ss.c +++ b/service-mgmt/sm/src/sm_failover_ss.c @@ -1,5 +1,5 @@ // -// Copyright (c) 2018 Wind River Systems, Inc. +// Copyright (c) 2018-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -268,10 +268,12 @@ SmErrorT _get_system_status(SmSystemStatusT& sys_status, char host_name[], char sys_status.host_status.mgmt_state = sm_failover_get_interface_info(SM_INTERFACE_MGMT); sys_status.host_status.cluster_host_state = sm_failover_get_interface_info(SM_INTERFACE_CLUSTER_HOST); sys_status.host_status.oam_state = sm_failover_get_interface_info(SM_INTERFACE_OAM); + sys_status.host_status.admin_state = sm_failover_get_interface_info(SM_INTERFACE_ADMIN); if(SM_FAILOVER_INTERFACE_OK == sys_status.host_status.mgmt_state || SM_FAILOVER_INTERFACE_OK == sys_status.host_status.oam_state || - SM_FAILOVER_INTERFACE_OK == sys_status.host_status.cluster_host_state) + SM_FAILOVER_INTERFACE_OK == sys_status.host_status.cluster_host_state || + SM_FAILOVER_INTERFACE_OK == sys_status.host_status.admin_state) { sys_status.heartbeat_state = SM_HEARTBEAT_OK; }else @@ -545,7 +547,9 @@ SmErrorT _get_survivor_dc(const SmSystemStatusT& system_status, SmSystemFailover { if(system_status.host_status.mgmt_state == SM_FAILOVER_INTERFACE_DOWN && (system_status.host_status.cluster_host_state == SM_FAILOVER_INTERFACE_DOWN || - system_status.host_status.cluster_host_state == SM_FAILOVER_INTERFACE_UNKNOWN)) + system_status.host_status.cluster_host_state == SM_FAILOVER_INTERFACE_UNKNOWN) && + (system_status.host_status.admin_state == SM_FAILOVER_INTERFACE_DOWN || + system_status.host_status.admin_state == SM_FAILOVER_INTERFACE_UNKNOWN)) { if(SM_FAILOVER_INTERFACE_DOWN == system_status.host_status.oam_state) { diff --git a/service-mgmt/sm/src/sm_failover_ss.h b/service-mgmt/sm/src/sm_failover_ss.h index f7ae09eb..2ca0f4f3 100644 --- a/service-mgmt/sm/src/sm_failover_ss.h +++ b/service-mgmt/sm/src/sm_failover_ss.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2018 Wind River Systems, Inc. +// Copyright (c) 2018-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -18,6 +18,7 @@ typedef struct SmFailoverInterfaceStateT mgmt_state; SmFailoverInterfaceStateT cluster_host_state; SmFailoverInterfaceStateT oam_state; + SmFailoverInterfaceStateT admin_state; SmNodeScheduleStateT current_schedule_state; }SmNodeStatusT; diff --git a/service-mgmt/sm/src/sm_heartbeat_thread.c b/service-mgmt/sm/src/sm_heartbeat_thread.c index 4ffe4989..7bb023ce 100644 --- a/service-mgmt/sm/src/sm_heartbeat_thread.c +++ b/service-mgmt/sm/src/sm_heartbeat_thread.c @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2017 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -225,7 +225,8 @@ SmErrorT sm_heartbeat_thread_add_interface( const SmServiceDomainInterfaceT& dom interface->multicast_socket = -1; interface->interface_type = sm_get_interface_type(domain_interface.service_domain_interface); - if ( SM_INTERFACE_OAM == interface->interface_type ) + if (( SM_INTERFACE_OAM == interface->interface_type ) || + ( SM_INTERFACE_ADMIN == interface->interface_type )) { interface->method = SEND_UNICAST; } else @@ -491,6 +492,11 @@ static bool sm_heartbeat_peer_alarm_on_interface( SmTimerIdT timer_id, { snprintf( network_type, sizeof(network_type), SM_CLUSTER_HOST_INTERFACE_NAME ); + } else if( 0 == strcmp( SM_SERVICE_DOMAIN_ADMIN_INTERFACE, + interface->service_domain_interface ) ) + { + snprintf( network_type, sizeof(network_type), SM_ADMIN_INTERFACE_NAME ); + } else { snprintf( network_type, sizeof(network_type), "unknown" ); } @@ -829,7 +835,8 @@ static bool sm_heartbeat_alive_timer( SmTimerIdT timer_id, int64_t user_data ) continue; } - if ( SM_INTERFACE_OAM == interface->interface_type ) + if (( SM_INTERFACE_OAM == interface->interface_type ) || + ( SM_INTERFACE_ADMIN == interface->interface_type )) { error = sm_heartbeat_msg_close_sockets( &(interface->unicast_socket) ); @@ -847,7 +854,8 @@ static bool sm_heartbeat_alive_timer( SmTimerIdT timer_id, int64_t user_data ) continue; } - if ( SM_INTERFACE_OAM == interface->interface_type ) + if (( SM_INTERFACE_OAM == interface->interface_type ) || + ( SM_INTERFACE_ADMIN == interface->interface_type )) { error = sm_heartbeat_msg_open_sockets( interface->network_type, &(interface->network_address), @@ -906,7 +914,8 @@ static bool sm_heartbeat_alive_timer( SmTimerIdT timer_id, int64_t user_data ) DPRINTFD( "Multicast not configured for interface %s", interface->interface_name ); } - if ( SM_INTERFACE_OAM == interface->interface_type ) + if (( SM_INTERFACE_OAM == interface->interface_type ) || + ( SM_INTERFACE_ADMIN == interface->interface_type )) { error = sm_heartbeat_msg_send_alive( interface->network_type, _node_name, &(interface->network_address), &(interface->network_peer_address), diff --git a/service-mgmt/sm/src/sm_main_event_handler.c b/service-mgmt/sm/src/sm_main_event_handler.c index 605d893c..422f38ff 100644 --- a/service-mgmt/sm/src/sm_main_event_handler.c +++ b/service-mgmt/sm/src/sm_main_event_handler.c @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2019 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -252,6 +252,26 @@ static void sm_main_event_handler_api_deprovision_service_callback( sm_deprovision_service(service_group_name, service_name); } +// **************************************************************************** +// Main Event Handler - Provision Service Domain Interface API Event Callback +// ====================================================== +static void sm_main_event_handler_api_provision_service_domain_interface_callback( + char service_domain_name[], char interface_name[], int seqno ) +{ + sm_provision_service_domain_interface(service_domain_name, interface_name); +} +// **************************************************************************** + +// **************************************************************************** +// Main Event Handler - Provision Service Domain Interface API Event Callback +// ====================================================== +static void sm_main_event_handler_api_deprovision_service_domain_interface_callback( + char service_domain_name[], char interface_name[], int seqno ) +{ + sm_deprovision_service_domain_interface(service_domain_name, interface_name); +} +// **************************************************************************** + // **************************************************************************** // Main Event Handler - Notify API Service Event Callback // ====================================================== @@ -382,6 +402,10 @@ SmErrorT sm_main_event_handler_initialize( void ) = sm_main_event_handler_api_provision_service_callback; _api_callbacks.deprovision_service = sm_main_event_handler_api_deprovision_service_callback; + _api_callbacks.provision_service_domain_interface + = sm_main_event_handler_api_provision_service_domain_interface_callback; + _api_callbacks.deprovision_service_domain_interface + = sm_main_event_handler_api_deprovision_service_domain_interface_callback; error = sm_api_register_callbacks( &_api_callbacks ); if( SM_OKAY != error ) diff --git a/service-mgmt/sm/src/sm_service_domain_interface_api.c b/service-mgmt/sm/src/sm_service_domain_interface_api.c index 948b4fb5..e4cb7fb9 100644 --- a/service-mgmt/sm/src/sm_service_domain_interface_api.c +++ b/service-mgmt/sm/src/sm_service_domain_interface_api.c @@ -1,5 +1,5 @@ // -// Copyright (c) 2014-2018 Wind River Systems, Inc. +// Copyright (c) 2014-2023 Wind River Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -85,7 +85,24 @@ static void sm_service_domain_interface_api_get_hw_interface( sm_error_str(error) ); return; } - } + } else if( 0 == strcmp( SM_SERVICE_DOMAIN_ADMIN_INTERFACE, + interface->service_domain_interface ) ) + { + error = sm_node_utils_get_admin_interface( interface->interface_name ); + if( SM_OKAY == error ) + { + DPRINTFI( "Network address (%s) maps to %s interface from config, " + "type=%s.", net_addr_str, interface->interface_name, + interface->service_domain_interface ); + goto PERSIST; + } + else if( SM_NOT_FOUND != error ) + { + DPRINTFE( "Failed to look up admin interface, error=%s.", + sm_error_str(error) ); + return; + } + } error = sm_hw_get_if_by_network_address( &(interface->network_address), interface->interface_name ); @@ -230,6 +247,46 @@ SmErrorT sm_service_domain_interface_api_node_disabled( void ) } // **************************************************************************** +// **************************************************************************** +// Service Domain Interface API - Interface Provisioned +// ============================================ +SmErrorT sm_service_domain_interface_api_provisioned( SmServiceDomainInterfaceT* interface ) +{ + char reason_text[SM_LOG_REASON_TEXT_MAX_CHAR]; + SmServiceDomainInterfaceEventT event; + void* user_data[] = { &event, reason_text }; + + event = SM_SERVICE_DOMAIN_INTERFACE_EVENT_UNKNOWN; + + snprintf( reason_text, sizeof(reason_text), "%s interface is enabled", + interface->service_domain_interface ); + + sm_service_domain_interface_api_send_event( user_data, interface ); + + return( SM_OKAY ); +} +// **************************************************************************** + +// **************************************************************************** +// Service Domain Interface API - Interface Deprovisioned +// ============================================ +SmErrorT sm_service_domain_interface_api_deprovisioned( SmServiceDomainInterfaceT* interface ) +{ + char reason_text[SM_LOG_REASON_TEXT_MAX_CHAR]; + SmServiceDomainInterfaceEventT event; + void* user_data[] = { &event, reason_text }; + + event = SM_SERVICE_DOMAIN_INTERFACE_EVENT_NOT_IN_USE; + + snprintf( reason_text, sizeof(reason_text), "%s interface is disabled", + interface->service_domain_interface ); + + sm_service_domain_interface_api_send_event( user_data, interface ); + + return( SM_OKAY ); +} +// **************************************************************************** + // **************************************************************************** // Service Domain Interface API - Audit // ==================================== diff --git a/service-mgmt/sm/src/sm_service_domain_interface_api.h b/service-mgmt/sm/src/sm_service_domain_interface_api.h index c8bc489b..460b537b 100644 --- a/service-mgmt/sm/src/sm_service_domain_interface_api.h +++ b/service-mgmt/sm/src/sm_service_domain_interface_api.h @@ -7,6 +7,7 @@ #define __SM_SERVICE_DOMAIN_INTERFACE_API_H__ #include "sm_types.h" +#include "sm_service_domain_interface_table.h" #ifdef __cplusplus extern "C" { @@ -30,6 +31,18 @@ extern SmErrorT sm_service_domain_interface_api_node_disabled( void ); extern SmErrorT sm_service_domain_interface_api_audit( void ); // **************************************************************************** +// **************************************************************************** +// Service Domain Interface API - Provision +// ==================================== +extern SmErrorT sm_service_domain_interface_api_provisioned( SmServiceDomainInterfaceT* interface ); +// **************************************************************************** + +// **************************************************************************** +// Service Domain Interface API - Deprovision +// ==================================== +extern SmErrorT sm_service_domain_interface_api_deprovisioned( SmServiceDomainInterfaceT* interface ); +// **************************************************************************** + // **************************************************************************** // Service Domain Interface API - Initialize // ========================================= diff --git a/service-mgmt/sm/src/sm_service_domain_interface_not_in_use_state.c b/service-mgmt/sm/src/sm_service_domain_interface_not_in_use_state.c index 25c850ea..2c57eb8b 100644 --- a/service-mgmt/sm/src/sm_service_domain_interface_not_in_use_state.c +++ b/service-mgmt/sm/src/sm_service_domain_interface_not_in_use_state.c @@ -12,6 +12,7 @@ #include "sm_debug.h" #include "sm_msg.h" #include "sm_log.h" +#include "sm_hw.h" #include "sm_service_domain_interface_fsm.h" // **************************************************************************** @@ -52,12 +53,49 @@ SmErrorT sm_service_domain_interface_niu_state_event_handler( SmServiceDomainInterfaceT* interface, SmServiceDomainInterfaceEventT event, void* event_data[] ) { + char reason_text[SM_LOG_REASON_TEXT_MAX_CHAR] = ""; + SmErrorT error; + bool enabled; + switch( event ) { case SM_SERVICE_DOMAIN_INTERFACE_EVENT_ENABLED: // undefined behavior break; + case SM_SERVICE_DOMAIN_INTERFACE_EVENT_UNKNOWN: + if (SM_SERVICE_DOMAIN_INTERFACE_EVENT_NOT_IN_USE != event) + { + error = sm_hw_get_if_state( interface->interface_name, &enabled ); + if( SM_OKAY != error ) + { + DPRINTFE( "Failed to audit hardware state of interface (%s), " + "error=%s", interface->interface_name, + sm_error_str( error ) ); + return( error ); + } + } + + snprintf( reason_text, sizeof(reason_text), "node and %s " + "enabled", interface->interface_name ); + + error = sm_service_domain_interface_fsm_set_state( + interface->service_domain, + interface->service_domain_interface, + SM_INTERFACE_STATE_UNKNOWN, + reason_text ); + if( SM_OKAY != error ) + { + DPRINTFE( "Set state (%s) of service domain (%s) interface " + "(%s) failed, error=%s.", + sm_interface_state_str( SM_INTERFACE_STATE_UNKNOWN ), + interface->service_domain, + interface->service_domain_interface, + sm_error_str( error ) ); + return( error ); + } + break; + case SM_SERVICE_DOMAIN_INTERFACE_EVENT_NODE_DISABLED: case SM_SERVICE_DOMAIN_INTERFACE_EVENT_DISABLED: case SM_SERVICE_DOMAIN_INTERFACE_EVENT_NODE_ENABLED: