Add PTHREAD_PRIO_PROTECT to sm mutexes

- rename mutexes from generic name '_mutex'
- create common util functions for initializing and destroying mutexes
- add mutex initialize/finalize functions
  and run them in sm_main_process_initialize
- update copyright info to 2023

Test plan:

PASS - AIO-DX: iso install
PASS - AIO-DX: verify ha swact
PASS - AIO-DX: failover swact test

PASS - AIO-DX: run pi_stress (rt-tests)
               to confirm priority inheritance POSIX attribute is working
-------------------------------------------------------------------
sysadmin@localhost:~$ sudo pi_stress --uniprocessor --duration=10s
Starting PI Stress Test
Number of thread groups: 3
Duration of test run: 10 seconds
Number of inversions per group: unlimited
     Admin thread SCHED_FIFO priority 4
3 groups of 3 threads will be created
      High thread SCHED_FIFO priority 3
       Med thread SCHED_FIFO priority 2
       Low thread SCHED_FIFO priority 1
Current Inversions: 992034
Stopping test
Total inversion performed: 992038
Test Duration: 0 days, 0 hours, 0 minutes, 11 seconds
-------------------------------------------------------------------

PASS - AIO-DX: valgrind helgrind
                * test for inconsistence lock ordering
                * test race conditions
                  [ detected outside scope of jira ]
                * test for deadlocks

Task: 47503
Story: 2010609

Signed-off-by: Kyale, Eliud <Eliud.Kyale@windriver.com>
Change-Id: Ic77f08ca7c3a687b1cc219ac9cba5711979206e8

service-mgmt/sm-common/src/sm_hw.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -25,6 +25,7 @@
 #include "sm_debug.h"
 #include "sm_selobj.h"
 #include "sm_netlink.h"
+#include "sm_util_types.h"
 
 typedef struct
 {
@@ -36,9 +37,19 @@ typedef struct
     SmHwCallbacksT callbacks;
 } SmHwThreadInfoT;
 
-static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t hw_mutex;
 static SmHwThreadInfoT _threads[SM_THREADS_MAX];
 
+SmErrorT sm_hw_mutex_initialize ( void )
+{
+    return sm_mutex_initialize(&hw_mutex, false);
+}
+
+SmErrorT sm_hw_mutex_finalize ( void )
+{
+    return sm_mutex_finalize(&hw_mutex);
+}
+
 // ****************************************************************************
 // Hardware - Find Thread Info
 // ===========================
@@ -270,7 +281,7 @@ SmErrorT sm_hw_get_if_state( const char if_name[], bool* enabled )
 SmErrorT sm_hw_get_all_qdisc_async( void )
 {
     struct nlmsghdr hdr;
-    struct tcmsg tc;    
+    struct tcmsg tc;
     SmHwThreadInfoT* thread_info;
     SmErrorT error;
 
@@ -361,7 +372,7 @@ static void sm_hw_netlink_if_msg_dispatch( SmHwThreadInfoT* thread_info,
         return;
     }
 
-    snprintf( if_change_data.interface_name, 
+    snprintf( if_change_data.interface_name,
               sizeof(if_change_data.interface_name), "%s",
               (char*) RTA_DATA( if_msg_attr[IFLA_IFNAME] ) );
 
@@ -601,7 +612,7 @@ static void sm_hw_netlink_qdisc_msg_dispatch( SmHwThreadInfoT* thread_info,
 // ****************************************************************************
 // Hardware - Netlink Message Dispatch
 // ===================================
-static void sm_hw_netlink_msg_dispatch( int socket_fd, 
+static void sm_hw_netlink_msg_dispatch( int socket_fd,
     struct sockaddr_nl* address, struct nlmsghdr* payload,
     void* invocation_data )
 {
@@ -673,7 +684,7 @@ SmErrorT sm_hw_initialize( SmHwCallbacksT* callbacks )
     SmHwThreadInfoT* thread_info = NULL;
     SmErrorT error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &hw_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -727,7 +738,7 @@ SmErrorT sm_hw_initialize( SmHwCallbacksT* callbacks )
     if( NULL != callbacks )
     {
         error = sm_netlink_open( &(thread_info->netlink_receive_socket),
-                                 RTMGRP_LINK | RTMGRP_IPV4_IFADDR | 
+                                 RTMGRP_LINK | RTMGRP_IPV4_IFADDR |
                                  RTMGRP_IPV6_IFADDR | RTMGRP_TC );
         if( SM_OKAY != error )
         {
@@ -757,7 +768,7 @@ SmErrorT sm_hw_initialize( SmHwCallbacksT* callbacks )
     thread_info->thread_id = pthread_self();
     thread_info->inuse = true;
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &hw_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -766,7 +777,7 @@ SmErrorT sm_hw_initialize( SmHwCallbacksT* callbacks )
     return( SM_OKAY );
 
 ERROR:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &hw_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -784,7 +795,7 @@ SmErrorT sm_hw_finalize( void )
     SmHwThreadInfoT* thread_info;
     SmErrorT error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &hw_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -817,7 +828,7 @@ SmErrorT sm_hw_finalize( void )
     }
 
 DONE:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &hw_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }

service-mgmt/sm-common/src/sm_hw.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -126,6 +126,18 @@ extern SmErrorT sm_hw_initialize( SmHwCallbacksT* callbacks );
 extern SmErrorT sm_hw_finalize( void );
 // ***************************************************************************
 
+// ****************************************************************************
+// initialize mutex
+// ==============
+extern SmErrorT sm_hw_mutex_initialize ( void );
+// ****************************************************************************
+
+// ****************************************************************************
+// destroy mutex
+// ==============
+extern SmErrorT sm_hw_mutex_finalize ( void );
+// ****************************************************************************
+
 #ifdef __cplusplus
 }
 #endif

service-mgmt/sm-common/src/sm_selobj.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -20,10 +20,11 @@
 #include "sm_types.h"
 #include "sm_debug.h"
 #include "sm_list.h"
+#include "sm_util_types.h"
 
 #define SM_SEL_OBJ_ENTRY_VALID  0xFDFDFDFD
 
-typedef struct 
+typedef struct
 {
     uint32_t valid;
     int selobj;
@@ -40,9 +41,19 @@ typedef struct
     SmSelObjSelectEntryT selobjs[SM_THREAD_SELECT_OBJS_MAX];
 } SmSelObjThreadInfoT;
 
-static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t selobj_mutex;
 static SmSelObjThreadInfoT _threads[SM_THREADS_MAX];
 
+SmErrorT sm_selobj_mutex_initialize ( void )
+{
+    return sm_mutex_initialize(&selobj_mutex, false);
+}
+
+SmErrorT sm_selobj_mutex_finalize ( void )
+{
+    return sm_mutex_finalize(&selobj_mutex);
+}
+
 // ****************************************************************************
 // Selection Object - Find Thread Info
 // ===================================
@@ -85,7 +96,7 @@ static SmSelObjSelectEntryT* sm_selobj_find( SmSelObjThreadInfoT* thread_info,
             }
         }
     }
-    
+
     return( NULL );
 }
 // ****************************************************************************
@@ -170,7 +181,7 @@ SmErrorT sm_selobj_deregister( int selobj )
                     thread_info->last_selobj = entry->selobj;
                 }
             }
-        }    
+        }
     }
 
     return( SM_OKAY );
@@ -198,11 +209,11 @@ SmErrorT sm_selobj_dispatch( unsigned int timeout_in_ms )
 
     num_fds = thread_info->last_selobj;
     fds = thread_info->selobjs_set;
-    
+
     tv.tv_sec = timeout_in_ms / 1000;
     tv.tv_usec = (timeout_in_ms % 1000) * 1000;
-        
+
-    result = select( num_fds+1, &fds, NULL, NULL, &tv ); 
+    result = select( num_fds+1, &fds, NULL, NULL, &tv );
     if( 0 > result )
     {
         if( errno == EINTR )
@@ -246,7 +257,7 @@ SmErrorT sm_selobj_initialize( void )
 {
     SmSelObjThreadInfoT* thread_info = NULL;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &selobj_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -260,21 +271,21 @@ SmErrorT sm_selobj_initialize( void )
             thread_info = &(_threads[thread_i]);
             break;
         }
-    }    
+    }
 
     if( NULL == thread_info )
     {
         DPRINTFE( "Failed to allocate thread information." );
         goto ERROR;
     }
-    
+
     memset( thread_info, 0, sizeof(SmSelObjThreadInfoT) );
 
     thread_info->inuse = true;
     thread_info->thread_id = pthread_self();
     FD_ZERO( &(thread_info->selobjs_set) );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &selobj_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
@@ -282,7 +293,7 @@ SmErrorT sm_selobj_initialize( void )
     return( SM_OKAY );
 
 ERROR:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &selobj_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -299,7 +310,7 @@ SmErrorT sm_selobj_finalize( void )
 {
     SmSelObjThreadInfoT* thread_info;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &selobj_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -315,7 +326,7 @@ SmErrorT sm_selobj_finalize( void )
     memset( thread_info, 0, sizeof(SmSelObjThreadInfoT) );
     FD_ZERO( &(thread_info->selobjs_set) );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &selobj_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }

service-mgmt/sm-common/src/sm_selobj.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -18,6 +18,18 @@ extern "C" {
 
 typedef void (*SmSelObjCallbackT) (int selobj, int64_t user_data);
 
+// ****************************************************************************
+// initialize mutex
+// ==============
+extern SmErrorT sm_selobj_mutex_initialize ( void );
+// ****************************************************************************
+
+// ****************************************************************************
+// destroy mutex
+// ==============
+extern SmErrorT sm_selobj_mutex_finalize ( void );
+// ****************************************************************************
+
 // ****************************************************************************
 // Selection Object - Register
 // ===========================

service-mgmt/sm-common/src/sm_thread_health.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -14,6 +14,7 @@
 #include "sm_types.h"
 #include "sm_debug.h"
 #include "sm_time.h"
+#include "sm_util_types.h"
 
 #define SM_THREAD_HEALTH_CHECK_INTERVAL_IN_MS   1000
 
@@ -23,11 +24,21 @@ typedef struct
     char thread_name[SM_THREAD_NAME_MAX_CHAR];
     long warn_after_elapsed_ms;
     long fail_after_elapsed_ms;
-    SmTimeT last_health_update; 
+    SmTimeT last_health_update;
 } SmThreadHealthT;
 
 static SmThreadHealthT _threads[SM_THREADS_MAX];
-static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t thread_health_mutex;
+
+SmErrorT sm_thread_health_mutex_initialize ( void )
+{
+    return sm_mutex_initialize(&thread_health_mutex, false);
+}
+
+SmErrorT sm_thread_health_mutex_finalize ( void)
+{
+    return sm_mutex_finalize(&thread_health_mutex);
+}
 
 // ****************************************************************************
 // Thread Health - Register
@@ -35,7 +46,7 @@ static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
 SmErrorT sm_thread_health_register( const char thread_name[],
     long warn_after_elapsed_ms, long fail_after_elapsed_ms )
 {
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &thread_health_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -60,7 +71,7 @@ SmErrorT sm_thread_health_register( const char thread_name[],
         }
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &thread_health_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -75,7 +86,7 @@ SmErrorT sm_thread_health_register( const char thread_name[],
 // ==========================
 SmErrorT sm_thread_health_deregister( const char thread_name[] )
 {
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &thread_health_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -98,7 +109,7 @@ SmErrorT sm_thread_health_deregister( const char thread_name[] )
         }
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &thread_health_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -113,7 +124,7 @@ SmErrorT sm_thread_health_deregister( const char thread_name[] )
 // ======================
 SmErrorT sm_thread_health_update( const char thread_name[] )
 {
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &thread_health_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -136,7 +147,7 @@ SmErrorT sm_thread_health_update( const char thread_name[] )
         }
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &thread_health_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -168,7 +179,7 @@ SmErrorT sm_thread_health_check( bool* healthy )
 
     sm_time_get( &last_check );
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &thread_health_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -199,7 +210,7 @@ SmErrorT sm_thread_health_check( bool* healthy )
         }
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &thread_health_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );

service-mgmt/sm-common/src/sm_thread_health.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -14,6 +14,18 @@
 extern "C" {
 #endif
 
+// ****************************************************************************
+// initialize mutex
+// ==============
+extern SmErrorT sm_thread_health_mutex_initialize ( void );
+// ****************************************************************************
+
+// ****************************************************************************
+// destroy mutex
+// ==============
+extern SmErrorT sm_thread_health_mutex_finalize ( void );
+// ****************************************************************************
+
 // ****************************************************************************
 // Thread Health - Register
 // ========================

service-mgmt/sm-common/src/sm_timer.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -23,6 +23,7 @@
 #include "sm_time.h"
 #include "sm_debug.h"
 #include "sm_selobj.h"
+#include "sm_util_types.h"
 
 #define SM_TIMER_ENTRY_INUSE                        0xFDFDFDFD
 #define SM_MAX_TIMERS                                      512
@@ -70,9 +71,19 @@ typedef struct
 
 static SmTimerIdT _next_timer_id = 1;
 
-static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t timer_mutex;
 static SmTimerThreadInfoT _threads[SM_THREADS_MAX];
 
+SmErrorT sm_timer_mutex_initialize ( void )
+{
+    return sm_mutex_initialize(&timer_mutex, false);
+}
+
+SmErrorT sm_timer_mutex_finalize ( void )
+{
+    return sm_mutex_finalize(&timer_mutex);
+}
+
 // ****************************************************************************
 // Timer - Find Thread Info
 // ========================
@@ -225,7 +236,7 @@ static SmErrorT sm_timer_schedule( SmTimerThreadInfoT* thread_info )
     for( timer_i=0; SM_MAX_TIMERS > timer_i; ++timer_i )
     {
         timer_entry = &(thread_info->timers[timer_i]);
-    
+
         if( SM_TIMER_ENTRY_INUSE == timer_entry->inuse )
         {
             ms_expired = sm_time_get_elapsed_ms( &timer_entry->arm_timestamp );
@@ -307,7 +318,7 @@ static void sm_timer_dispatch( int selobj, int64_t user_data )
         DPRINTFE( "Failed to find thread information." );
         return;
     }
-    
+
     ms_expired = sm_time_get_elapsed_ms( &thread_info->sched_timestamp );
     if( ms_expired >= (thread_info->tick_interval_in_ms*SM_MAX_TICK_INTERVALS) )
     {
@@ -342,13 +353,13 @@ static void sm_timer_dispatch( int selobj, int64_t user_data )
         if( SM_TIMER_ENTRY_INUSE == timer_entry->inuse )
         {
             ms_expired = sm_time_get_elapsed_ms( &timer_entry->arm_timestamp );
- 
+
             if( ms_expired >= timer_entry->ms_interval )
             {
                 bool rearm;
                 uint64_t timer_instance = timer_entry->timer_instance;
                 SmTimerHistoryEntryT* history_entry;
-                
+
                 history_entry = &(history[total_timers_fired]);
 
                 history_entry->inuse = SM_TIMER_ENTRY_INUSE;
@@ -542,7 +553,7 @@ SmErrorT sm_timer_initialize( unsigned int tick_interval_in_ms )
     SmTimerThreadInfoT* thread_info = NULL;
     SmErrorT error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &timer_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -577,7 +588,7 @@ SmErrorT sm_timer_initialize( unsigned int tick_interval_in_ms )
 
     if( 0 > thread_info->tick_timer_fd )
     {
-        DPRINTFE( "Failed to create tick timer, error=%s.", 
+        DPRINTFE( "Failed to create tick timer, error=%s.",
                   strerror( errno ) );
         goto ERROR;
     }
@@ -610,7 +621,7 @@ SmErrorT sm_timer_initialize( unsigned int tick_interval_in_ms )
         goto ERROR;
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &timer_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -619,7 +630,7 @@ SmErrorT sm_timer_initialize( unsigned int tick_interval_in_ms )
     return( SM_OKAY );
 
 ERROR:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &timer_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -637,7 +648,7 @@ SmErrorT sm_timer_finalize( void )
     SmTimerThreadInfoT* thread_info;
     SmErrorT error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &timer_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -665,7 +676,7 @@ SmErrorT sm_timer_finalize( void )
     }
 
 DONE:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &timer_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }

service-mgmt/sm-common/src/sm_timer.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -18,12 +18,24 @@
 extern "C" {
 #endif
 
-#define SM_TIMER_ID_INVALID -1 
+#define SM_TIMER_ID_INVALID -1
 
 typedef int64_t SmTimerIdT;
 
 typedef bool (*SmTimerCallbackT) (SmTimerIdT timer_id, int64_t user_data );
 
+// ****************************************************************************
+// initialize mutex
+// ==============
+extern SmErrorT sm_timer_mutex_initialize ( void );
+// ****************************************************************************
+
+// ****************************************************************************
+// destroy mutex
+// ==============
+extern SmErrorT sm_timer_mutex_finalize ( void );
+// ****************************************************************************
+
 // ****************************************************************************
 // Timer - Register
 // ================

service-mgmt/sm-common/src/sm_trap.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -25,6 +25,7 @@
 #include "sm_types.h"
 #include "sm_debug.h"
 #include "sm_trap_thread.h"
+#include "sm_util_types.h"
 
 typedef struct
 {
@@ -37,7 +38,7 @@ static int _trap_fd = -1;
 static int _process_id;
 static char _process_name[SM_PROCESS_NAME_MAX_CHAR];
 static SmTrapThreadInfoT _threads[SM_THREADS_MAX];
-static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t trap_mutex;
 static pthread_spinlock_t _thread_spinlock;
 
 // ***************************************************************************
@@ -158,12 +159,12 @@ void sm_trap_set_thread_info( void )
     int thread_id;
     SmTrapThreadInfoT* info;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &trap_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return;
     }
-    
+
     thread_id = (int) syscall( SYS_gettid );
 
     info = sm_trap_find_thread_info( thread_id );
@@ -185,7 +186,7 @@ void sm_trap_set_thread_info( void )
         }
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &trap_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return;
@@ -207,6 +208,8 @@ SmErrorT sm_trap_initialize( const char process_name[] )
     int dummy_trace_count;
     SmErrorT error = SM_FAILED;
 
+    sm_mutex_initialize(&trap_mutex, false);
+
     _process_id = (int) getpid();
     snprintf( _process_name, sizeof(_process_name), "%s", process_name );
 
@@ -258,9 +261,9 @@ SmErrorT sm_trap_initialize( const char process_name[] )
             printf( "Failed to set flags, error=%s.\n", strerror( errno ) );
             error = SM_FAILED;
             goto ERROR;
-        }            
+        }
     }
-    
+
     error = sm_trap_thread_start( trap_fds[0] );
     if( SM_OKAY != error )
     {
@@ -312,9 +315,7 @@ ERROR:
 // ===============
 SmErrorT sm_trap_finalize( void )
 {
-    SmErrorT error;
+    SmErrorT error = sm_trap_thread_stop();
-
-    error = sm_trap_thread_stop();
     if( SM_OKAY != error )
     {
         printf( "Failed to stop trap thread, error=%s.\n",
@@ -327,6 +328,8 @@ SmErrorT sm_trap_finalize( void )
         _trap_fd = -1;
     }
 
+    sm_mutex_finalize(&trap_mutex);
+
     return( SM_OKAY );
 }
 // ***************************************************************************

service-mgmt/sm-common/src/sm_util_types.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -8,10 +8,57 @@
 #include "sm_util_types.h"
 #include "sm_debug.h"
 
+SmErrorT sm_mutex_initialize ( pthread_mutex_t* mutex, bool mutex_recursive )
+{
+    SmErrorT error = SM_OKAY;
+    pthread_mutexattr_t attr;
+    if(mutex)
+    {
+        pthread_mutexattr_init(&attr);
+        if(mutex_recursive)
+        {
+            pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
+        }
+        pthread_mutexattr_setprotocol( &attr, PTHREAD_PRIO_INHERIT);
+        int res = pthread_mutex_init(mutex, &attr);
+        if( res != 0 )
+        {
+            DPRINTFE("Failed to initialize mutex, error %d", res);
+            error = SM_FAILED;
+        }
+    }
+    else
+    {
+        DPRINTFE("Failed to initialize mutex, NULL ptr");
+        error = SM_FAILED;
+    }
+    return error;
+}
+
+SmErrorT sm_mutex_finalize ( pthread_mutex_t* mutex )
+{
+    SmErrorT error = SM_OKAY;
+    if(mutex)
+    {
+        int res = pthread_mutex_destroy(mutex);
+        if( res != 0 )
+        {
+            DPRINTFE("Failed to destroy mutex, error %d", res);
+            error = SM_FAILED;
+        }
+    }
+    else
+    {
+        DPRINTFE("Failed to destroy mutex, NULL ptr");
+        error = SM_FAILED;
+    }
+    return error;
+}
+
 mutex_holder::mutex_holder(pthread_mutex_t* mutex)
 {
-    this->_mutex = mutex;
+    this->m_mutex = mutex;
-    if( 0 != pthread_mutex_lock( this->_mutex ) )
+    if( 0 != pthread_mutex_lock( this->m_mutex ) )
     {
         DPRINTFE("Critical error, failed to obtain the mutex. Quit...");
         abort();
@@ -20,7 +67,7 @@ mutex_holder::mutex_holder(pthread_mutex_t* mutex)
 
 mutex_holder::~mutex_holder()
 {
-    if( 0 != pthread_mutex_unlock( this->_mutex ) )
+    if( 0 != pthread_mutex_unlock( this->m_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }

service-mgmt/sm-common/src/sm_util_types.h

@@ -1,20 +1,34 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
 #ifndef __SM_UTIL_TYPES_H__
 #define __SM_UTIL_TYPES_H__
 
+#include <stdbool.h>
 #include <pthread.h>
+#include "sm_types.h"
 
 class mutex_holder
 {
     private:
-        pthread_mutex_t* _mutex;
+        pthread_mutex_t* m_mutex;
     public:
         mutex_holder(pthread_mutex_t* mutex);
         virtual ~mutex_holder();
 };
 
+// ****************************************************************************
+// initialize mutex
+// ==============
+SmErrorT sm_mutex_initialize ( pthread_mutex_t* mutex, bool mutex_recursive );
+// ****************************************************************************
+
+// ****************************************************************************
+// destroy mutex
+// ==============
+SmErrorT sm_mutex_finalize ( pthread_mutex_t* mutex );
+// ****************************************************************************
+
 #endif //__SM_UTIL_TYPES_H__

service-mgmt/sm/src/fm_api_wrapper.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -8,25 +8,36 @@
 #include <pthread.h>
 
 #include "sm_debug.h"
+#include "sm_util_types.h"
 
-static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t fm_api_mutex;
+
+SmErrorT fm_mutex_initialize ( void )
+{
+    return sm_mutex_initialize(&fm_api_mutex, false);
+}
+
+SmErrorT fm_mutex_finalize ( void )
+{
+    return sm_mutex_finalize(&fm_api_mutex);
+}
 
 // ****************************************************************************
 // FM Wrapper - FM Set Fault
 // =========================
 EFmErrorT fm_set_fault_wrapper( const SFmAlarmDataT* alarm, fm_uuid_t* uuid )
 {
-    EFmErrorT fm_error; 
+    EFmErrorT fm_error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
-        return( FM_ERR_NOCONNECT ); 
+        return( FM_ERR_NOCONNECT );
     }
 
     fm_error = fm_set_fault( alarm, uuid );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
@@ -40,17 +51,17 @@ EFmErrorT fm_set_fault_wrapper( const SFmAlarmDataT* alarm, fm_uuid_t* uuid )
 // ===========================
 EFmErrorT fm_clear_fault_wrapper( AlarmFilter* filter )
 {
-    EFmErrorT fm_error; 
+    EFmErrorT fm_error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
-        return( FM_ERR_NOCONNECT ); 
+        return( FM_ERR_NOCONNECT );
     }
 
     fm_error = fm_clear_fault( filter );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
@@ -64,17 +75,17 @@ EFmErrorT fm_clear_fault_wrapper( AlarmFilter* filter )
 // =========================
 EFmErrorT fm_clear_all_wrapper( fm_ent_inst_t* inst_id )
 {
-    EFmErrorT fm_error; 
+    EFmErrorT fm_error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
-        return( FM_ERR_NOCONNECT ); 
+        return( FM_ERR_NOCONNECT );
     }
 
     fm_error = fm_clear_all( inst_id );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
@@ -88,17 +99,17 @@ EFmErrorT fm_clear_all_wrapper( fm_ent_inst_t* inst_id )
 // =========================
 EFmErrorT fm_get_fault_wrapper( AlarmFilter* filter, SFmAlarmDataT* alarm )
 {
-    EFmErrorT fm_error; 
+    EFmErrorT fm_error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
-        return( FM_ERR_NOCONNECT ); 
+        return( FM_ERR_NOCONNECT );
     }
 
     fm_error = fm_get_fault( filter, alarm );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
@@ -113,17 +124,17 @@ EFmErrorT fm_get_fault_wrapper( AlarmFilter* filter, SFmAlarmDataT* alarm )
 EFmErrorT fm_get_faults_wrapper( fm_ent_inst_t* inst_id, SFmAlarmDataT* alarm,
     unsigned int* max_alarms_to_get )
 {
-    EFmErrorT fm_error; 
+    EFmErrorT fm_error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
-        return( FM_ERR_NOCONNECT ); 
+        return( FM_ERR_NOCONNECT );
     }
 
     fm_error = fm_get_faults( inst_id, alarm, max_alarms_to_get );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
@@ -135,20 +146,20 @@ EFmErrorT fm_get_faults_wrapper( fm_ent_inst_t* inst_id, SFmAlarmDataT* alarm,
 // ****************************************************************************
 // FM Wrapper - FM Get Faults By Id
 // ================================
-EFmErrorT fm_get_faults_by_id_wrapper( fm_alarm_id* alarm_id, 
+EFmErrorT fm_get_faults_by_id_wrapper( fm_alarm_id* alarm_id,
     SFmAlarmDataT* alarm, unsigned int* max_alarms_to_get )
 {
-    EFmErrorT fm_error; 
+    EFmErrorT fm_error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
-        return( FM_ERR_NOCONNECT ); 
+        return( FM_ERR_NOCONNECT );
     }
 
     fm_error = fm_get_faults_by_id( alarm_id, alarm, max_alarms_to_get );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &fm_api_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }

service-mgmt/sm/src/fm_api_wrapper.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -8,6 +8,7 @@
 
 
 #include <fmAPI.h>
+#include "sm_types.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -17,7 +18,7 @@ extern "C" {
 // The following functions are wrapped to guarantee thread safety of the
 // FM API library.
 //
-extern EFmErrorT fm_set_fault_wrapper( const SFmAlarmDataT* alarm, 
+extern EFmErrorT fm_set_fault_wrapper( const SFmAlarmDataT* alarm,
     fm_uuid_t* uuid);
 
 extern EFmErrorT fm_clear_fault_wrapper( AlarmFilter *filter );
@@ -27,10 +28,14 @@ extern EFmErrorT fm_clear_all_wrapper( fm_ent_inst_t *inst_id );
 extern EFmErrorT fm_get_fault_wrapper( AlarmFilter *filter,
     SFmAlarmDataT* alarm );
 
+extern SmErrorT fm_mutex_initialize ( void );
+
+extern SmErrorT fm_mutex_finalize ( void );
+
 EFmErrorT fm_get_faults_wrapper( fm_ent_inst_t *inst_id,
     SFmAlarmDataT *alarm, unsigned int* max_alarms_to_get );
 
-EFmErrorT fm_get_faults_by_id_wrapper( fm_alarm_id* alarm_id, 
+EFmErrorT fm_get_faults_by_id_wrapper( fm_alarm_id* alarm_id,
         SFmAlarmDataT* alarm, unsigned int* max_alarms_to_get );
 
 #ifdef __cplusplus

service-mgmt/sm/src/sm_alarm.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -11,7 +11,7 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
-#include <time.h> 
+#include <time.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <errno.h>
@@ -26,6 +26,7 @@
 #include "sm_time.h"
 #include "sm_node_utils.h"
 #include "sm_alarm_thread.h"
+#include "sm_util_types.h"
 
 #define SM_ALARM_ENTRY_INUSE                                         0xA5A5A5A5
 
@@ -51,7 +52,7 @@ typedef struct
 
 static int _server_fd = -1;
 static int _client_fd = -1;
-static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t alarm_mutex;
 static SmAlarmEntryT _alarms[SM_ALARMS_MAX];
 
 // ****************************************************************************
@@ -77,7 +78,7 @@ static SmAlarmEntryT* sm_alarm_find_empty( void )
 // ****************************************************************************
 // Alarm - Find
 // ============
-static SmAlarmEntryT* sm_alarm_find( const SmAlarmT alarm, 
+static SmAlarmEntryT* sm_alarm_find( const SmAlarmT alarm,
     const SmAlarmNodeNameT node_name, const SmAlarmDomainNameT domain_name,
     const SmAlarmEntityNameT entity_name )
 {
@@ -112,7 +113,7 @@ static bool sm_alarm_throttle( const SmAlarmT alarm, SmAlarmStateT alarm_state,
     bool throttle = false;
     SmAlarmEntryT* entry;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &alarm_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         abort();
@@ -129,7 +130,7 @@ static bool sm_alarm_throttle( const SmAlarmT alarm, SmAlarmStateT alarm_state,
         }
 
         memset( entry, 0, sizeof(SmAlarmEntryT) );
-    
+
         entry->inuse = SM_ALARM_ENTRY_INUSE;
         entry->alarm = alarm;
         snprintf( entry->alarm_node_name, sizeof(entry->alarm_node_name),
@@ -167,7 +168,7 @@ static bool sm_alarm_throttle( const SmAlarmT alarm, SmAlarmStateT alarm_state,
     }
 
 RELEASE_MUTEX:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &alarm_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         abort();
@@ -210,7 +211,7 @@ bool sm_alarm_write_alarm_additional_text( const char data[],
 // ****************************************************************************
 // Alarm - Raise
 // =============
-static void sm_alarm_raise( const SmAlarmT alarm, 
+static void sm_alarm_raise( const SmAlarmT alarm,
     const SmAlarmNodeNameT node_name, const SmAlarmDomainNameT domain_name,
     const SmAlarmEntityNameT entity_name, SmAlarmDataT* data )
 {
@@ -251,7 +252,7 @@ static void sm_alarm_raise( const SmAlarmT alarm,
     if( 0 > result )
     {
         DPRINTFE( "Failed to raise alarm (%i) for domain (%s) entity (%s), "
-                  "error=%s.", alarm, domain_name, entity_name, 
+                  "error=%s.", alarm, domain_name, entity_name,
                   strerror( errno ) );
     }
 }
@@ -292,7 +293,7 @@ void sm_alarm_raise_state_alarm( const SmAlarmT alarm,
               "%s", entity_condition );
 
     data.threshold_info.applicable = false;
-    
+
     if( '\0' == proposed_repair_action[0] )
     {
         snprintf( data.proposed_repair_action,
@@ -554,7 +555,7 @@ void sm_alarm_clear( const SmAlarmT alarm, const SmAlarmNodeNameT node_name,
     if( 0 > result )
     {
         DPRINTFE( "Failed to clear alarm (%i) for domain (%s) entity (%s), "
-                  "error=%s.", alarm, domain_name, entity_name, 
+                  "error=%s.", alarm, domain_name, entity_name,
                   strerror( errno ) );
     }
 }
@@ -569,7 +570,7 @@ void sm_alarm_clear_all( const SmAlarmDomainNameT domain_name )
     SmAlarmThreadMsgClearAllAlarmsT* clear_all_alarms = &(msg.u.clear_all_alarms);
     int result;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &alarm_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         abort();
@@ -577,7 +578,7 @@ void sm_alarm_clear_all( const SmAlarmDomainNameT domain_name )
 
     memset( _alarms, 0, sizeof(_alarms) );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &alarm_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         abort();
@@ -647,7 +648,9 @@ SmErrorT sm_alarm_initialize( void )
     DPRINTFD( "Alarm message size is %i bytes.",
               (int) sizeof(SmAlarmThreadMsgT) );
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    sm_mutex_initialize(&alarm_mutex, false);
+
+    if( 0 != pthread_mutex_lock( &alarm_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         abort();
@@ -655,12 +658,12 @@ SmErrorT sm_alarm_initialize( void )
 
     memset( _alarms, 0, sizeof(_alarms) );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &alarm_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         abort();
     }
- 
+
     result = socketpair( AF_UNIX, SOCK_DGRAM, 0, sockets );
     if( 0 > result )
     {
@@ -684,7 +687,7 @@ SmErrorT sm_alarm_initialize( void )
         if( 0 > result )
         {
             DPRINTFE( "Failed to set alarm communication socket (%i) to "
-                      "non-blocking, error=%s.", socket_i, 
+                      "non-blocking, error=%s.", socket_i,
                       strerror( errno ) );
             return( SM_FAILED );
         }
@@ -720,7 +723,7 @@ SmErrorT sm_alarm_initialize( void )
                   sm_error_str( error ) );
         return( error );
     }
-    
+
     return( SM_OKAY );
 }
 // ****************************************************************************
@@ -731,7 +734,7 @@ SmErrorT sm_alarm_initialize( void )
 SmErrorT sm_alarm_finalize( void )
 {
     SmErrorT error;
-    
+
     error = sm_alarm_thread_stop();
     if( SM_OKAY != error )
     {
@@ -751,7 +754,7 @@ SmErrorT sm_alarm_finalize( void )
         _client_fd = -1;
     }
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &alarm_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         abort();
@@ -759,12 +762,14 @@ SmErrorT sm_alarm_finalize( void )
 
     memset( _alarms, 0, sizeof(_alarms) );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &alarm_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         abort();
     }
-    
+
+    sm_mutex_finalize(&alarm_mutex);
+
     return( SM_OKAY );
 }
 // ****************************************************************************

service-mgmt/sm/src/sm_cluster_hbs_info_msg.cpp

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -23,6 +23,7 @@
 #include "sm_selobj.h"
 #include "sm_worker_thread.h"
 #include "sm_node_utils.h"
+#include "sm_util_types.h"
 
 // uncomment when debugging this module to enabled DPRINTFD output to log file
 // #define __DEBUG__MSG__
@@ -117,7 +118,7 @@ void log_cluster_hbs_state(const SmClusterHbsStateT& state)
     }
 }
 
-pthread_mutex_t SmClusterHbsInfoMsg::_mutex;
+pthread_mutex_t SmClusterHbsInfoMsg::sm_cluster_hbs_mutex;
 const unsigned short Invalid_Req_Id = 0;
 int SmClusterHbsInfoMsg::_sock = -1;
 SmClusterHbsStateT SmClusterHbsInfoMsg::_cluster_hbs_state_current;
@@ -228,7 +229,7 @@ bool SmClusterHbsInfoMsg::_process_cluster_hbs_history(mtce_hbs_cluster_history_
 void SmClusterHbsInfoMsg::_cluster_hbs_info_msg_received( int selobj, int64_t user_data )
 {
     mtce_hbs_cluster_type msg = {0};
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_cluster_hbs_mutex);
     while(true)
     {
         int bytes_read = recv( selobj, &msg, sizeof(msg), MSG_NOSIGNAL | MSG_DONTWAIT );
@@ -349,7 +350,7 @@ bool SmClusterHbsInfoMsg::cluster_hbs_info_query(cluster_hbs_query_ready_callbac
     unsigned short reqid;
     struct timespec ts;
     {
-        mutex_holder holder(&_mutex);
+        mutex_holder holder(&sm_cluster_hbs_mutex);
         if(alive_pulse)
         {
             reqid = 0;
@@ -473,14 +474,10 @@ SmErrorT SmClusterHbsInfoMsg::open_socket()
 SmErrorT SmClusterHbsInfoMsg::initialize()
 {
     SmErrorT error;
-    pthread_mutexattr_t attr;
 
-    pthread_mutexattr_init(&attr);
+    error = sm_mutex_initialize(&sm_cluster_hbs_mutex, true);
-    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
+    if( error != SM_OKAY )
-    int res = pthread_mutex_init(&_mutex, &attr);
-    if( 0 != res )
     {
-        DPRINTFE("Failed to initialize mutex, error %d", res);
         return SM_FAILED;
     }
 
@@ -530,13 +527,15 @@ SmErrorT SmClusterHbsInfoMsg::initialize()
 
 SmErrorT SmClusterHbsInfoMsg::finalize()
 {
-    mutex_holder holder(&_mutex);
-    if(_sock > 0)
     {
-        close(_sock);
+        mutex_holder holder(&sm_cluster_hbs_mutex);
-        _sock = -1;
+        if(_sock > 0)
+        {
+            close(_sock);
+            _sock = -1;
+        }
     }
-    pthread_mutex_destroy(&_mutex);
+    pthread_mutex_destroy(&sm_cluster_hbs_mutex);
     return SM_OKAY;
 }
 

service-mgmt/sm/src/sm_cluster_hbs_info_msg.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -86,7 +86,7 @@ class SmClusterHbsInfoMsg
     private:
         static int _sock;
         static unsigned short _last_reqid;
-        static pthread_mutex_t _mutex;
+        static pthread_mutex_t sm_cluster_hbs_mutex;
         static SmClusterHbsStateT _cluster_hbs_state_current;
         static SmClusterHbsStateT _cluster_hbs_state_previous;
         static hbs_query_respond_callback _callbacks;
@@ -104,4 +104,4 @@ class SmClusterHbsInfoMsg
         static int this_controller_index;
 };
 
-#endif // __SM_CLUSTER_HBS_INFO_MSG_H__
+#endif // __SM_CLUSTER_HBS_INFO_MSG_H__

service-mgmt/sm/src/sm_failover.c

@@ -139,7 +139,7 @@ static SmFailoverInterfaceInfo* _mgmt_interface_info = NULL;
 static SmFailoverInterfaceInfo* _cluster_host_interface_info = NULL;
 static SmFailoverInterfaceInfo* _admin_interface_info = NULL;
 static SmFailoverInterfaceInfo _peer_if_list[SM_INTERFACE_MAX];
-static pthread_mutex_t _mutex;
+static pthread_mutex_t sm_failover_mutex;
 static SmDbHandleT* _sm_db_handle = NULL;
 
 static SmNodeScheduleStateT _host_state;
@@ -160,6 +160,16 @@ SmErrorT sm_exec_json_command(const char* cmd, char result_buf[], int result_len
 SmErrorT sm_failover_get_node_oper_state(char* node_name, SmNodeOperationalStateT *state);
 void _log_nodes_state();
 
+SmErrorT sm_failover_mutex_initialize ( void )
+{
+    return sm_mutex_initialize(&sm_failover_mutex, true);
+}
+
+SmErrorT sm_failover_mutex_finalize ( void )
+{
+    return sm_mutex_finalize(&sm_failover_mutex);
+}
+
 // ****************************************************************************
 // Failover - interface check
 // ==================
@@ -254,7 +264,7 @@ static SmFailoverInterfaceInfo* find_interface_info( SmFailoverInterfaceT* inter
 // ==================
 void sm_failover_lost_hello_msg()
 {
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_failover_mutex);
 
     if(_hello_msg_alive)
     {
@@ -268,7 +278,7 @@ void sm_failover_lost_hello_msg()
 // ==================
 void sm_failover_hello_msg_restore()
 {
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_failover_mutex);
 
     SmFailoverInterfaceInfo* iter;
     for(iter = _my_if_list; iter < _end_of_list; iter ++)
@@ -287,7 +297,7 @@ void sm_failover_hello_msg_restore()
 // ==================
 void sm_failover_lost_heartbeat( SmFailoverInterfaceT* interface )
 {
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_failover_mutex);
 
     SmFailoverInterfaceInfo* if_info = find_interface_info( interface );
     if ( NULL == if_info )
@@ -319,7 +329,7 @@ void sm_failover_lost_heartbeat( SmFailoverInterfaceT* interface )
 // ==================
 void sm_failover_heartbeat_restore( SmFailoverInterfaceT* interface )
 {
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_failover_mutex);
 
     SmFailoverInterfaceInfo* if_info = find_interface_info( interface );
     if ( NULL == if_info )
@@ -364,7 +374,7 @@ void sm_failover_heartbeat_restore( SmFailoverInterfaceT* interface )
 // ==================
 void sm_failover_interface_down( const char* const interface_name )
 {
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_failover_mutex);
 
     SmFailoverInterfaceInfo* iter;
     int impacted = 0;
@@ -398,7 +408,7 @@ void sm_failover_interface_down( const char* const interface_name )
 // ==================
 void sm_failover_interface_up( const char* const interface_name )
 {
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_failover_mutex);
 
     SmFailoverInterfaceInfo* iter;
     int impacted = 0;
@@ -468,7 +478,7 @@ void sm_failover_if_state_update(const char node_name[], SmHeartbeatMsgIfStateT
 {
     if( 0 == strcmp(node_name, _peer_name) )
     {
-        mutex_holder holder(&_mutex);
+        mutex_holder holder(&sm_failover_mutex);
 
         if( _peer_if_state != if_state )
         {
@@ -575,7 +585,7 @@ int sm_failover_get_if_state()
 // ==================
 SmHeartbeatMsgIfStateT sm_failover_if_state_get()
 {
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_failover_mutex);
     int if_state_flag = sm_failover_get_if_state();
     return (if_state_flag & 0b0111); //the lower 3 bits i/f state flag
 }
@@ -586,7 +596,7 @@ SmHeartbeatMsgIfStateT sm_failover_if_state_get()
 // ==================
 SmHeartbeatMsgIfStateT sm_failover_get_peer_if_state()
 {
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_failover_mutex);
 
     return (_peer_if_state & 0b0111); //the lower 3 bits i/f state flag
 }
@@ -993,7 +1003,7 @@ SmErrorT sm_failover_set_system(const SmSystemFailoverStatus& failover_status)
 // =======================
 void sm_failover_audit()
 {
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_failover_mutex);
 
     timespec now;
     time_t now_ms;
@@ -1291,17 +1301,6 @@ SmErrorT sm_failover_initialize( void )
     bool enabled;
     SmErrorT error;
 
-    pthread_mutexattr_t attr;
-
-    pthread_mutexattr_init(&attr);
-    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
-    int res = pthread_mutex_init(&_mutex, &attr);
-    if( 0 != res )
-    {
-        DPRINTFE("Failed to initialize mutex, error %d", res);
-        return SM_FAILED;
-    }
-
     error = SmFailoverFSM::initialize();
     if( SM_OKAY != error )
     {
@@ -1460,7 +1459,6 @@ SmErrorT sm_failover_finalize( void )
         DPRINTFE( "Failed to finalize failover FSM, error %s.", sm_error_str( error ) );
     }
 
-    pthread_mutex_destroy(&_mutex);
     return SM_OKAY;
 }
 // ****************************************************************************

service-mgmt/sm/src/sm_failover.h

@@ -40,6 +40,18 @@ typedef enum
     SM_FAILOVER_PEER_DISABLED = 0x4000,
 }SmFailoverCommFaultBitFlagT;
 
+// ****************************************************************************
+// initialize mutex
+// ==============
+extern SmErrorT sm_failover_mutex_initialize ( void );
+// ****************************************************************************
+
+// ****************************************************************************
+// destroy mutex
+// ==============
+extern SmErrorT sm_failover_mutex_finalize ( void );
+// ****************************************************************************
+
 // ****************************************************************************
 // Failover - check
 // ==================

service-mgmt/sm/src/sm_heartbeat_thread.c

@@ -10,7 +10,7 @@
 #include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
-#include <time.h> 
+#include <time.h>
 #include <unistd.h>
 #include <signal.h>
 #include <errno.h>
@@ -36,6 +36,7 @@
 #include "sm_log.h"
 #include "sm_failover.h"
 #include "sm_cluster_hbs_info_msg.h"
+#include "sm_util_types.h"
 
 #define SM_HEARTBEAT_THREAD_NAME                                    "sm_heartbeat"
 #define SM_HEARTBEAT_THREAD_TICK_INTERVAL_IN_MS                                100
@@ -96,7 +97,7 @@ typedef struct
 
 static sig_atomic_t _stay_on;
 static bool _thread_created = false;
-static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t heartbeat_mutex;
 static pthread_t _heartbeat_thread;
 static sig_atomic_t _messaging_enabled = 0;
 static sig_atomic_t _heartbeat_required = 1;
@@ -129,6 +130,17 @@ static bool operator != ( const SmNetworkAddressT& src, const SmNetworkAddressT&
 }
 // ****************************************************************************
 
+SmErrorT sm_heartbeat_thread_mutex_initialize ( void )
+{
+    return sm_mutex_initialize(&heartbeat_mutex, false);
+}
+
+SmErrorT sm_heartbeat_thread_mutex_finalize ( void )
+{
+    return sm_mutex_finalize(&heartbeat_mutex);
+}
+
+
 // ****************************************************************************
 // Heartbeat Thread - Disable heartbeat
 // =========================
@@ -200,7 +212,7 @@ SmErrorT sm_heartbeat_thread_add_interface( const SmServiceDomainInterfaceT& dom
     SmHeartbeatThreadInterfaceT* interface;
     SmErrorT error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -248,7 +260,7 @@ SmErrorT sm_heartbeat_thread_add_interface( const SmServiceDomainInterfaceT& dom
            ( interface->network_peer_port != domain_interface.network_peer_heartbeat_port ))
         {
             // Same id, but not the same instance data.
-            error = sm_heartbeat_msg_close_sockets( 
+            error = sm_heartbeat_msg_close_sockets(
                                         &(interface->multicast_socket) );
             if( SM_OKAY != error )
             {
@@ -307,7 +319,7 @@ SmErrorT sm_heartbeat_thread_add_interface( const SmServiceDomainInterfaceT& dom
                   interface->interface_name );
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -316,7 +328,7 @@ SmErrorT sm_heartbeat_thread_add_interface( const SmServiceDomainInterfaceT& dom
     return( SM_OKAY );
 
 ERROR:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -333,7 +345,7 @@ SmErrorT sm_heartbeat_thread_delete_interface( int64_t id )
 {
     SmHeartbeatThreadInterfaceT* interface;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -347,7 +359,7 @@ SmErrorT sm_heartbeat_thread_delete_interface( int64_t id )
         free( interface );
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -442,7 +454,7 @@ static bool sm_heartbeat_peer_alarm_on_interface( SmTimerIdT timer_id,
         return( true );
     }
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( true );
@@ -579,7 +591,7 @@ static bool sm_heartbeat_peer_alarm_on_interface( SmTimerIdT timer_id,
         sm_log_communication_state_change( network_type, peer_interface->log_text );
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
@@ -587,19 +599,19 @@ static bool sm_heartbeat_peer_alarm_on_interface( SmTimerIdT timer_id,
     return( true );
 
 ERROR:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
-    } 
+    }
 
     return( rearm );
 }
 // ****************************************************************************
 
 // ****************************************************************************
-// Heartbeat Thread - Add Peer Interface 
+// Heartbeat Thread - Add Peer Interface
 // =====================================
-SmErrorT sm_heartbeat_thread_add_peer_interface( int64_t id, 
+SmErrorT sm_heartbeat_thread_add_peer_interface( int64_t id,
     char interface_name[], SmNetworkAddressT* network_address,
     int network_port, int dead_interval )
 {
@@ -613,7 +625,7 @@ SmErrorT sm_heartbeat_thread_add_peer_interface( int64_t id,
         return( SM_OKAY );
     }
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -664,7 +676,7 @@ SmErrorT sm_heartbeat_thread_add_peer_interface( int64_t id,
                          (SmListEntryDataPtrT) peer_interface );
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -673,7 +685,7 @@ SmErrorT sm_heartbeat_thread_add_peer_interface( int64_t id,
     return( SM_OKAY );
 
 ERROR:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -684,7 +696,7 @@ ERROR:
 // ****************************************************************************
 
 // ****************************************************************************
-// Heartbeat Thread - Delete Peer Interface 
+// Heartbeat Thread - Delete Peer Interface
 // ========================================
 SmErrorT sm_heartbeat_thread_delete_peer_interface( char interface_name[],
     SmNetworkAddressT* network_address, int network_port )
@@ -692,7 +704,7 @@ SmErrorT sm_heartbeat_thread_delete_peer_interface( char interface_name[],
     SmHeartbeatThreadPeerInterfaceT* peer_interface;
     SmErrorT error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -722,7 +734,7 @@ SmErrorT sm_heartbeat_thread_delete_peer_interface( char interface_name[],
         free( peer_interface );
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( SM_FAILED );
@@ -759,13 +771,13 @@ static SmHeartbeatThreadPeerNodeT* sm_heartbeat_thread_find_peer_node(
 // ****************************************************************************
 // Heartbeat Thread - Peer Alive In Period
 // =======================================
-bool sm_heartbeat_thread_peer_alive_in_period( char node_name[], 
+bool sm_heartbeat_thread_peer_alive_in_period( char node_name[],
     unsigned int period_in_ms )
 {
     bool peer_alive = false;
     SmHeartbeatThreadPeerNodeT* peer_node;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( true );
@@ -781,7 +793,7 @@ bool sm_heartbeat_thread_peer_alive_in_period( char node_name[],
         peer_alive = ( period_in_ms >= ms_expired );
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
@@ -800,7 +812,7 @@ static bool sm_heartbeat_alive_timer( SmTimerIdT timer_id, int64_t user_data )
     SmHeartbeatThreadInterfaceT* interface;
     SmErrorT error;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( true );
@@ -965,7 +977,7 @@ static bool sm_heartbeat_alive_timer( SmTimerIdT timer_id, int64_t user_data )
     }
 
 DONE:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( true );
@@ -987,7 +999,7 @@ static bool sm_heartbeat_thread_auth_message( char interface_name[],
     SmHeartbeatThreadInterfaceT* interface;
     SmSha512HashT hash;
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( true );
@@ -1020,7 +1032,7 @@ static bool sm_heartbeat_thread_auth_message( char interface_name[],
 
 
 DONE:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return( true );
@@ -1041,7 +1053,7 @@ static void sm_heartbeat_thread_receive_alive_message( char node_name[],
 
     DPRINTFD( "Received alive message from node (%s).", node_name );
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return;
@@ -1095,7 +1107,7 @@ static void sm_heartbeat_thread_receive_alive_message( char node_name[],
     }
 
 DONE:
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
         return;

service-mgmt/sm/src/sm_heartbeat_thread.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014-2017 Wind River Systems, Inc.
+// Copyright (c) 2014-2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -16,6 +16,18 @@
 extern "C" {
 #endif
 
+// ****************************************************************************
+// initialize mutex
+// ==============
+extern SmErrorT sm_heartbeat_thread_mutex_initialize ( void );
+// ****************************************************************************
+
+// ****************************************************************************
+// destroy mutex
+// ==============
+extern SmErrorT sm_heartbeat_thread_mutex_finalize ( void );
+// ****************************************************************************
+
 // ****************************************************************************
 // Heartbeat Thread - Disable heartbeat
 // =========================
@@ -50,7 +62,7 @@ extern SmErrorT sm_heartbeat_thread_delete_interface( int64_t id );
 // ****************************************************************************
 
 // ****************************************************************************
-// Heartbeat Thread - Add Peer Interface 
+// Heartbeat Thread - Add Peer Interface
 // =====================================
 extern SmErrorT sm_heartbeat_thread_add_peer_interface( int64_t id,
     char interface_name[], SmNetworkAddressT* network_address,
@@ -58,7 +70,7 @@ extern SmErrorT sm_heartbeat_thread_add_peer_interface( int64_t id,
 // ****************************************************************************
 
 // ****************************************************************************
-// Heartbeat Thread - Delete Peer Interface 
+// Heartbeat Thread - Delete Peer Interface
 // ========================================
 extern SmErrorT sm_heartbeat_thread_delete_peer_interface(
     char interface_name[], SmNetworkAddressT* network_address,

service-mgmt/sm/src/sm_process.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -43,6 +43,7 @@
 #include "sm_service_group_api.h"
 #include "sm_service_api.h"
 #include "sm_service_action.h"
+#include "sm_service_enable.h"
 #include "sm_service_heartbeat_api.h"
 #include "sm_service_heartbeat_thread.h"
 #include "sm_service_domain_scheduler.h"
@@ -55,6 +56,8 @@
 #include "sm_worker_thread.h"
 #include "sm_configuration_table.h"
 #include "sm_cluster_hbs_info_msg.h"
+#include "fm_api_wrapper.h"
+#include "sm_swact_state.h"
 
 #define SM_PROCESS_DB_CHECKPOINT_INTERVAL_IN_MS         30000
 #define SM_PROCESS_TICK_INTERVAL_IN_MS                    200
@@ -170,6 +173,18 @@ static SmErrorT sm_process_initialize( void )
 {
     SmErrorT error;
 
+    // mutexes
+    fm_mutex_initialize();
+    sm_failover_mutex_initialize();
+    sm_service_enable_mutex_initialize();
+    sm_swact_state_mutex_initialize();
+    sm_service_heartbeat_api_mutex_initialize();
+    sm_heartbeat_thread_mutex_initialize();
+    sm_thread_health_mutex_initialize();
+    sm_selobj_mutex_initialize();
+    sm_timer_mutex_initialize();
+    sm_hw_mutex_initialize();
+
     error = sm_selobj_initialize();
     if( SM_OKAY != error )
     {
@@ -538,6 +553,18 @@ static SmErrorT sm_process_finalize( void )
                   sm_error_str( error ) );
     }
 
+    // mutexes
+    fm_mutex_finalize();
+    sm_failover_mutex_finalize();
+    sm_service_enable_mutex_finalize();
+    sm_swact_state_mutex_finalize();
+    sm_service_heartbeat_api_mutex_finalize();
+    sm_heartbeat_thread_mutex_finalize();
+    sm_thread_health_mutex_finalize();
+    sm_selobj_mutex_finalize();
+    sm_timer_mutex_finalize();
+    sm_hw_mutex_finalize();
+
     return( SM_OKAY );
 }
 // ****************************************************************************

service-mgmt/sm/src/sm_service_enable.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -29,7 +29,7 @@
 #define EXTRA_CORES_STORAGE "/etc/platform/.task_affining_incomplete"
 #define PLATFORM_CORES_FILE "/var/run/sm/.platform_cores"
 
-static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t sm_service_enable_mutex;
 static sem_t sem;
 static int num_initial_cores = 0;
 static sem_t sem_more_cores;
@@ -37,13 +37,22 @@ static bool check_more_cores_available = false;
 static bool more_cores_available = false;
 static int  num_extra_cores = 0;
 
+SmErrorT sm_service_enable_mutex_initialize ( void )
+{
+    return sm_mutex_initialize(&sm_service_enable_mutex, false);
+}
+
+SmErrorT sm_service_enable_mutex_finalize ( void )
+{
+    return sm_mutex_finalize(&sm_service_enable_mutex);
+}
 
 // ****************************************************************************
 // add more cores for enabling service
 // ==============
 void sm_service_enable_throttle_add_cores( bool additional_cores )
 {
-    mutex_holder holder(&_mutex);
+    mutex_holder holder(&sm_service_enable_mutex);
 
     check_more_cores_available = additional_cores;
     if (!additional_cores)
@@ -187,7 +196,7 @@ REPORT:
 static bool sm_service_enable_timeout( SmTimerIdT timer_id, int64_t user_data )
 {
     int64_t id = user_data;
-    SmServiceT* service;    
+    SmServiceT* service;
     SmServiceActionT action_running;
     SmServiceActionResultT action_result;
     SmServiceStateT service_state;
@@ -236,7 +245,7 @@ static bool sm_service_enable_timeout( SmTimerIdT timer_id, int64_t user_data )
 
     DPRINTFI( "Action (%s) timeout with result (%s), state (%s), "
               "status (%s), and condition (%s) for service (%s), "
-              "reason_text=%s, exit_code=%i.", 
+              "reason_text=%s, exit_code=%i.",
               sm_service_action_str( action_running ),
               sm_service_action_result_str( action_result ),
               sm_service_state_str( service_state ),
@@ -276,7 +285,7 @@ static bool sm_service_enable_timeout( SmTimerIdT timer_id, int64_t user_data )
 static void sm_service_enable_complete( pid_t pid, int exit_code,
     int64_t user_data )
 {
-    SmServiceT* service;    
+    SmServiceT* service;
     SmServiceActionT action_running;
     SmServiceActionResultT action_result;
     SmServiceStateT service_state;
@@ -297,7 +306,7 @@ static void sm_service_enable_complete( pid_t pid, int exit_code,
                   (int) pid, sm_error_str(SM_NOT_FOUND) );
         return;
     }
-    
+
     sm_service_enable_throttle_uncheck();
 
     if( SM_SERVICE_ACTION_ENABLE != service->action_running )
@@ -319,7 +328,7 @@ static void sm_service_enable_complete( pid_t pid, int exit_code,
                       "service (%s), error=%s.",
                       sm_service_action_str( service->action_running ),
                       service->name, sm_error_str( error ) );
-        } 
+        }
     }
 
     action_running = service->action_running;
@@ -335,7 +344,7 @@ static void sm_service_enable_complete( pid_t pid, int exit_code,
         service_condition = SM_SERVICE_CONDITION_UNKNOWN;
 
     } else {
-        error = sm_service_action_result( exit_code, service->name, 
+        error = sm_service_action_result( exit_code, service->name,
                                           service->action_running,
                                           &action_result, &service_state,
                                           &service_status, &service_condition,
@@ -354,7 +363,7 @@ static void sm_service_enable_complete( pid_t pid, int exit_code,
 
     DPRINTFI( "Action (%s) completed with result (%s), state (%s), "
               "status (%s), and condition (%s) for service (%s), "
-              "reason_text=%s, exit_code=%i.", 
+              "reason_text=%s, exit_code=%i.",
               sm_service_action_str( action_running ),
               sm_service_action_result_str( action_result ),
               sm_service_state_str( service_state ),
@@ -412,7 +421,7 @@ SmErrorT sm_service_enable( SmServiceT* service )
     // Run action.
     error = sm_service_action_run( service->name,
                                    service->instance_name,
-                                   service->instance_params,            
+                                   service->instance_params,
                                    SM_SERVICE_ACTION_ENABLE,
                                    &process_id, &timeout );
     if( SM_OKAY != error )
@@ -421,7 +430,7 @@ SmErrorT sm_service_enable( SmServiceT* service )
                   "error=%s.", service->name, sm_error_str( error ) );
         return( error );
     }
-   
+
     // Register for action script exit notification.
     error = sm_process_death_register( process_id, true,
                                        sm_service_enable_complete, 0 );
@@ -661,7 +670,7 @@ bool sm_service_enable_throttle_check( void )
         return true;
     } else if( EAGAIN == errno )
     {
-        mutex_holder holder(&_mutex);
+        mutex_holder holder(&sm_service_enable_mutex);
         if(check_more_cores_available)
         {
             int new_num_platform_cores = get_extra_cores();

service-mgmt/sm/src/sm_service_enable.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -13,6 +13,18 @@
 extern "C" {
 #endif
 
+// ****************************************************************************
+// initialize mutex
+// ==============
+extern SmErrorT sm_service_enable_mutex_initialize ( void );
+// ****************************************************************************
+
+// ****************************************************************************
+// destroy mutex
+// ==============
+extern SmErrorT sm_service_enable_mutex_finalize ( void );
+// ****************************************************************************
+
 // ****************************************************************************
 // add more cores for enabling service
 // ==============

service-mgmt/sm/src/sm_service_heartbeat_api.c

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -19,6 +19,7 @@
 #include "sm_list.h"
 #include "sm_selobj.h"
 #include "sm_timer.h"
+#include "sm_util_types.h"
 
 typedef enum
 {
@@ -79,15 +80,25 @@ typedef struct
 static int _sm_heartbeat_api_server_fd = -1;
 static int _sm_heartbeat_api_client_fd = -1;
 static SmListT* _callbacks = NULL;
-static pthread_mutex_t _mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t heartbeat_callback_mutex;
+
+SmErrorT sm_service_heartbeat_api_mutex_initialize ( void )
+{
+    return sm_mutex_initialize(&heartbeat_callback_mutex, true); // PTHREAD_MUTEX_RECURSIVE
+}
+
+SmErrorT sm_service_heartbeat_api_mutex_finalize ( void )
+{
+    return sm_mutex_finalize(&heartbeat_callback_mutex);
+}
 
 // ****************************************************************************
 // Service Heartbeat API - Register Callbacks
 // ==========================================
-SmErrorT sm_service_heartbeat_api_register_callbacks( 
+SmErrorT sm_service_heartbeat_api_register_callbacks(
     SmServiceHeartbeatCallbacksT* callbacks )
 {
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_callback_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -95,7 +106,7 @@ SmErrorT sm_service_heartbeat_api_register_callbacks(
 
     SM_LIST_PREPEND( _callbacks, (SmListEntryDataPtrT) callbacks );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_callback_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
@@ -110,7 +121,7 @@ SmErrorT sm_service_heartbeat_api_register_callbacks(
 SmErrorT sm_service_heartbeat_api_deregister_callbacks(
     SmServiceHeartbeatCallbacksT* callbacks )
 {
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_callback_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return( SM_FAILED );
@@ -118,11 +129,11 @@ SmErrorT sm_service_heartbeat_api_deregister_callbacks(
 
     SM_LIST_REMOVE( _callbacks, (SmListEntryDataPtrT) callbacks );
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_callback_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
-    
+
     return( SM_OKAY );
 }
 // ****************************************************************************
@@ -296,7 +307,7 @@ static void sm_service_heartbeat_api_dispatch( int selobj, int64_t user_data )
             break;
 
         } else if( 0 == bytes_read ) {
-            // For connection oriented sockets, this indicates that the peer 
+            // For connection oriented sockets, this indicates that the peer
             // has performed an orderly shutdown.
             return;
 
@@ -317,7 +328,7 @@ static void sm_service_heartbeat_api_dispatch( int selobj, int64_t user_data )
         return;
     }
 
-    if( 0 != pthread_mutex_lock( &_mutex ) )
+    if( 0 != pthread_mutex_lock( &heartbeat_callback_mutex ) )
     {
         DPRINTFE( "Failed to capture mutex." );
         return;
@@ -372,13 +383,13 @@ static void sm_service_heartbeat_api_dispatch( int selobj, int64_t user_data )
             break;
 
             default:
-                DPRINTFE( "Unknown message type (%i) received.", 
+                DPRINTFE( "Unknown message type (%i) received.",
                           msg.msg_type );
             break;
         }
     }
 
-    if( 0 != pthread_mutex_unlock( &_mutex ) )
+    if( 0 != pthread_mutex_unlock( &heartbeat_callback_mutex ) )
     {
         DPRINTFE( "Failed to release mutex." );
     }
@@ -421,7 +432,7 @@ SmErrorT sm_service_heartbeat_api_initialize( bool main_process )
             if( 0 > result )
             {
                 DPRINTFE( "Failed to set service heartbeat socket (%i) to "
-                          "non-blocking, error=%s.", socket_i, 
+                          "non-blocking, error=%s.", socket_i,
                           strerror( errno ) );
                 return( SM_FAILED );
             }
@@ -430,7 +441,7 @@ SmErrorT sm_service_heartbeat_api_initialize( bool main_process )
         _sm_heartbeat_api_server_fd = sockets[0];
         _sm_heartbeat_api_client_fd = sockets[1];
 
-        error = sm_selobj_register( _sm_heartbeat_api_client_fd, 
+        error = sm_selobj_register( _sm_heartbeat_api_client_fd,
                                     sm_service_heartbeat_api_dispatch, 0 );
         if( SM_OKAY != error )
         {
@@ -439,7 +450,7 @@ SmErrorT sm_service_heartbeat_api_initialize( bool main_process )
             return( error );
         }
     } else {
-        error = sm_selobj_register( _sm_heartbeat_api_server_fd, 
+        error = sm_selobj_register( _sm_heartbeat_api_server_fd,
                                     sm_service_heartbeat_api_dispatch, 0 );
         if( SM_OKAY != error )
         {

service-mgmt/sm/src/sm_service_heartbeat_api.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2014 Wind River Systems, Inc.
+// Copyright (c) 2014,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -31,17 +31,29 @@ typedef struct
     SmServiceHeartbeatFailCallbackT fail_callback;
 } SmServiceHeartbeatCallbacksT;
 
+// ****************************************************************************
+// initialize mutex
+// ==============
+extern SmErrorT sm_service_heartbeat_api_mutex_initialize ( void );
+// ****************************************************************************
+
+// ****************************************************************************
+// destroy mutex
+// ==============
+extern SmErrorT sm_service_heartbeat_api_mutex_finalize ( void );
+// ****************************************************************************
+
 // ****************************************************************************
 // Service Heartbeat API - Register Callbacks
 // ==========================================
-extern SmErrorT sm_service_heartbeat_api_register_callbacks( 
+extern SmErrorT sm_service_heartbeat_api_register_callbacks(
     SmServiceHeartbeatCallbacksT* callbacks );
 // ****************************************************************************
 
 // ****************************************************************************
 // Service Heartbeat API - Deregister Callbacks
 // ============================================
-extern SmErrorT sm_service_heartbeat_api_deregister_callbacks( 
+extern SmErrorT sm_service_heartbeat_api_deregister_callbacks(
     SmServiceHeartbeatCallbacksT* callbacks );
 // ****************************************************************************
 

service-mgmt/sm/src/sm_swact_state.c

@@ -1,12 +1,24 @@
-// Copyright (c) 2017 Wind River Systems, Inc.
+// Copyright (c) 2017,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
 #include "sm_swact_state.h"
 #include "pthread.h"
+#include "sm_debug.h"
+#include "sm_util_types.h"
 
 volatile SmSwactStateT _swact_state = SM_SWACT_STATE_NONE;
-static pthread_mutex_t _state_mutex;
+static pthread_mutex_t swact_state_mutex;
+
+SmErrorT sm_swact_state_mutex_initialize ( void )
+{
+    return sm_mutex_initialize(&swact_state_mutex, false);
+}
+
+SmErrorT sm_swact_state_mutex_finalize ( void )
+{
+    return sm_mutex_finalize(&swact_state_mutex);
+}
 
 // ****************************************************************************
 // Swact State - Swact State Setter
@@ -14,9 +26,9 @@ static pthread_mutex_t _state_mutex;
 void sm_set_swact_state(SmSwactStateT state)
 {
     // The state is set by sm main thread and task affining thread.
-    pthread_mutex_lock(&_state_mutex);
+    pthread_mutex_lock(&swact_state_mutex);
      _swact_state = state;
-    pthread_mutex_unlock(&_state_mutex);
+    pthread_mutex_unlock(&swact_state_mutex);
 }
 
 // ****************************************************************************

service-mgmt/sm/src/sm_swact_state.h

@@ -1,11 +1,13 @@
 //
-// Copyright (c) 2017 Wind River Systems, Inc.
+// Copyright (c) 2017,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
 #ifndef __SM_SWACT_STATE_H__
 #define __SM_SWACT_STATE_H__
 
+#include "sm_types.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -17,6 +19,18 @@ typedef enum
     SM_SWACT_STATE_END,
 } SmSwactStateT;
 
+// ****************************************************************************
+// initialize mutex
+// ==============
+extern SmErrorT sm_swact_state_mutex_initialize ( void );
+// ****************************************************************************
+
+// ****************************************************************************
+// destroy mutex
+// ==============
+extern SmErrorT sm_swact_state_mutex_finalize ( void );
+// ****************************************************************************
+
 // ****************************************************************************
 // Swact State - Setter
 // ==========================================

service-mgmt/sm/src/sm_worker_thread.cpp

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -73,7 +73,7 @@ SmErrorT SmWorkerThread::finalize()
 // ****************************************************************************
 SmWorkerThread::SmWorkerThread() : _priority_queue(), _regular_queue()
 {
-    this->_mutex = PTHREAD_MUTEX_INITIALIZER;
+    sm_mutex_initialize(&this->sm_action_mutex, false);
     this->_goon = true;
     this->_thread_created = false;
 }
@@ -85,6 +85,7 @@ SmWorkerThread::SmWorkerThread() : _priority_queue(), _regular_queue()
 SmWorkerThread::~SmWorkerThread()
 {
     sem_destroy(&this->_sem);
+    sm_mutex_finalize(&this->sm_action_mutex);
 }
 // ****************************************************************************
 
@@ -157,7 +158,7 @@ SmErrorT SmWorkerThread::stop()
 // ****************************************************************************
 void SmWorkerThread::add_action(SmAction* action)
 {
-    mutex_holder(&this->_mutex);
+    mutex_holder(&this->sm_action_mutex);
     this->_regular_queue.push(action);
     int res = sem_post(&_sem);
     if(0 != res)
@@ -174,7 +175,7 @@ void SmWorkerThread::add_action(SmAction* action)
 // ****************************************************************************
 void SmWorkerThread::add_priority_action(SmAction* action)
 {
-    mutex_holder(&this->_mutex);
+    mutex_holder(&this->sm_action_mutex);
     this->_priority_queue.push(action);
     int res = sem_post(&_sem);
     if(0 != res)

service-mgmt/sm/src/sm_worker_thread.h

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2018 Wind River Systems, Inc.
+// Copyright (c) 2018,2023 Wind River Systems, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -66,7 +66,7 @@ class SmWorkerThread
         static SmErrorT finalize();
 
     private:
-        pthread_mutex_t _mutex;
+        pthread_mutex_t sm_action_mutex;
         SmActionQueueT _priority_queue;
         SmActionQueueT _regular_queue;
         sem_t _sem;