starlingx
/
distcloud
Code Issues Proposed changes

Fix up confirm_stop functions of DC OCF scripts 

The DC OCF scripts were not updated over the switch to Debian
in StarlingX 8.0. As a result, it could lead to orphan processes
over the service restart or controller swact. The orphan processes
consume resources and perform duplicate/obsolete tasks (e.g.
auditing the same subclouds as the corresponding worker processes)
until their work queues are empty.

This commit fixes up the pgrep option to restore the functionality
of the confirm_stop function of the OCF script. Processes that
fail to be terminated will get killed.

Test Plan:
  - Deploy a small DC system. Verify that all DC services can
    be started, stopped and restarted by SM.
  - Deploy a large DC system with many subclouds. Reduce the
    thread_pool_size of dcmanager-audit-worker. Let the system
    soak for a couple of hours. Restart the service in the
    middle of the audit cycle. Verify that dcmanager-audit-worker
    sevice was successfully restarted and there are no orphan
    processes.

Closes-Bug: 2064368
Change-Id: Ie5cbc89cde374e32d4e0a3799a9f8833c071d206
Signed-off-by: Tee Ngo <tee.ngo@windriver.com>
This commit is contained in:
Li Zhu 2024-04-30 22:43:30 -04:00 committed by Tee Ngo
parent c98abd9af9
commit 7ce8b72869
11 changed files with 30 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
distributedcloud/ocf/dcdbsync-api
View File

@ -4,7 +4,7 @@
# Description: Manages a DC Orchestrator DBsync API Service
# (dcdbsync-api) process as an HA resource
#
# Copyright (c) 2019 Wind River Systems, Inc.
# Copyright (c) 2019,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -234,12 +234,12 @@ dcdbsync_api_confirm_stop() {
local my_processes
my_binary=`which ${OCF_RESKEY_binary}`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then
ocf_log info "About to SIGKILL the following: ${my_processes}"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"
fi
}

6
distributedcloud/ocf/dcmanager-api
View File

@ -3,7 +3,7 @@
#
# Description: Manages an OpenStack DC Manager API Service (dcmanager-api) process as an HA resource
#
# Copyright (c) 2017 Wind River Systems, Inc.
# Copyright (c) 2017,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -233,12 +233,12 @@ dcmanager_api_confirm_stop() {
local my_processes
my_binary=`which ${OCF_RESKEY_binary}`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then
ocf_log info "About to SIGKILL the following: ${my_processes}"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"
fi
}

6
distributedcloud/ocf/dcmanager-audit
View File

@ -5,7 +5,7 @@
# Manages an OpenStack DC Manager Audit Service (dcmanager-audit)
# process as an HA resource
#
# Copyright (c) 2020 Wind River Systems, Inc.
# Copyright (c) 2020,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -233,12 +233,12 @@ dcmanager_audit_confirm_stop() {
local my_processes
my_binary=`which ${OCF_RESKEY_binary}`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then
ocf_log info "About to SIGKILL the following: ${my_processes}"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"
fi
}

6
distributedcloud/ocf/dcmanager-audit-worker
View File

@ -5,7 +5,7 @@
# Manages an OpenStack DC Manager Audit-Worker Service (dcmanager-audit-worker)
# process as an HA resource
#
# Copyright (c) 2021 Wind River Systems, Inc.
# Copyright (c) 2021,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -233,12 +233,12 @@ dcmanager_audit_confirm_stop() {
local my_processes
my_binary=`which ${OCF_RESKEY_binary}`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then
ocf_log info "About to SIGKILL the following: ${my_processes}"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"
fi
}

6
distributedcloud/ocf/dcmanager-manager
View File

@ -5,7 +5,7 @@
# Manages an OpenStack DC Manager Service (dcmanager-manager)
# process as an HA resource
#
# Copyright (c) 2017 Wind River Systems, Inc.
# Copyright (c) 2017,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -237,12 +237,12 @@ dcmanager_manager_confirm_stop() {
local my_processes
my_binary=`which ${OCF_RESKEY_binary}`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then
ocf_log info "About to SIGKILL the following: ${my_processes}"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"
fi
}

6
distributedcloud/ocf/dcmanager-orchestrator
View File

@ -5,7 +5,7 @@
# Manages an OpenStack DC Manager Orchestrator Service (dcmanager-orchestrator)
# process as an HA resource
#
# Copyright (c) 2020 Wind River Systems, Inc.
# Copyright (c) 2020,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -233,12 +233,12 @@ dcmanager_orchestrator_confirm_stop() {
local my_processes
my_binary=`which ${OCF_RESKEY_binary}`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then
ocf_log info "About to SIGKILL the following: ${my_processes}"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"
fi
}

6
distributedcloud/ocf/dcmanager-state
View File

@ -5,7 +5,7 @@
# Manages an OpenStack DC Manager State Service (dcmanager-state)
# process as an HA resource
#
# Copyright (c) 2022 Wind River Systems, Inc.
# Copyright (c) 2022,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -237,12 +237,12 @@ dcmanager_state_confirm_stop() {
local my_processes
my_binary=`which ${OCF_RESKEY_binary}`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then
ocf_log info "About to SIGKILL the following: ${my_processes}"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"
fi
}

6
distributedcloud/ocf/dcorch-engine
View File

@ -3,7 +3,7 @@
#
# Description: Manages an OpenStack DC Orchestrator Engine Service (dcorch-engine) process as an HA resource
#
# Copyright (c) 2017 Wind River Systems, Inc.
# Copyright (c) 2017,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -233,12 +233,12 @@ dcorch_engine_confirm_stop() {
local my_processes
my_binary=`which ${OCF_RESKEY_binary}`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"`
my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then
ocf_log info "About to SIGKILL the following: ${my_processes}"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"
pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary}([^\w-]|$)"
fi
}

4
distributedcloud/ocf/dcorch-identity-api-proxy
View File

@ -6,7 +6,7 @@
# process as an HA resource
#
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -236,7 +236,7 @@ dcorch_identity_api_proxy_confirm_stop() {
my_binary=`which ${OCF_RESKEY_binary}`
my_type="identity"
my_processes=`pgrep -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary} .*--type ${my_type}([^\w-]|$)"`
my_processes=`pgrep -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary} .*--type ${my_type}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then

4
distributedcloud/ocf/dcorch-patch-api-proxy
View File

@ -5,7 +5,7 @@
# Manages an OpenStack DC Orchestrator Patching Api Proxy Service (dcorch-patch-api-proxy)
# process as an HA resource
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -236,7 +236,7 @@ dcorch_patch_api_proxy_confirm_stop() {
my_binary=`which ${OCF_RESKEY_binary}`
my_type="patching"
my_processes=`pgrep -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary} .*--type ${my_type}([^\w-]|$)"`
my_processes=`pgrep -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary} .*--type ${my_type}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then

4
distributedcloud/ocf/dcorch-sysinv-api-proxy
View File

@ -5,7 +5,7 @@
# Manages an OpenStack DC Orchestrator System Inventory Api Proxy Service (dcorch-sysinv-api-proxy)
# process as an HA resource
#
# Copyright (c) 2017 Wind River Systems, Inc.
# Copyright (c) 2017,2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -236,7 +236,7 @@ dcorch_sysinv_api_proxy_confirm_stop() {
my_binary=`which ${OCF_RESKEY_binary}`
my_type="platform"
my_processes=`pgrep -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary} .*--type ${my_type}([^\w-]|$)"`
my_processes=`pgrep -f "^(python|/usr/bin/python|/usr/bin/python3) ${my_binary} .*--type ${my_type}([^\w-]|$)"`
if [ -n "${my_processes}" ]
then