From a85ffc695ed7a1f42f39ecfe0f76e54db958389a Mon Sep 17 00:00:00 2001 From: Bin Qian Date: Thu, 13 Apr 2023 16:44:05 +0000 Subject: [PATCH] Shorten rabbit failure recovery delay In rare cases, when system running slowly with significant scheduling delay, rabbit disable action timeout continually. As final resort sm reboots the impacted controller for recovery after failure count reaches MAX_TRANSITION_FAILURES. As rabbit service disable timeout is set to 60 seconds, this result a significant delay before reboot for recovery. This change updates MAX_TRANSITION_FAILURES of rabbit service from 16 to 5 to reduce the delay of recovery of rabbit failure. TCs passed: Install a DX system Observed service group recovery escalated to reboot after 5 forced rabbit disable failure. Closes-bug: 2016168 Signed-off-by: Bin Qian Change-Id: I660a64f0e78b6564456eb26245b672d2549f9a3b --- service-mgmt/sm-db/database/create_sm_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service-mgmt/sm-db/database/create_sm_db.sql b/service-mgmt/sm-db/database/create_sm_db.sql index 014495fa..ea429af0 100644 --- a/service-mgmt/sm-db/database/create_sm_db.sql +++ b/service-mgmt/sm-db/database/create_sm_db.sql @@ -112,7 +112,7 @@ INSERT INTO "SERVICES" VALUES(8,'yes','rabbit-fs','initial','initial','none','no INSERT INTO "SERVICES" VALUES(9,'yes','nfs-mgmt','initial','initial','none','none',2,1,90000,4,16,''); INSERT INTO "SERVICES" VALUES(11,'yes','platform-fs','initial','initial','none','none',2,1,90000,4,16,''); INSERT INTO "SERVICES" VALUES(12,'yes','postgres','initial','initial','none','none',2,1,90000,4,16,'/var/run/postmaster.pid'); -INSERT INTO "SERVICES" VALUES(13,'yes','rabbit','initial','initial','none','none',2,1,90000,4,16,'/var/run/rabbitmq/rabbitmq.pid'); +INSERT INTO "SERVICES" VALUES(13,'yes','rabbit','initial','initial','none','none',2,1,90000,4,5,'/var/run/rabbitmq/rabbitmq.pid'); INSERT INTO "SERVICES" VALUES(17,'yes','platform-nfs-ip','initial','initial','none','none',2,1,90000,4,16,''); INSERT INTO "SERVICES" VALUES(15,'yes','platform-export-fs','initial','initial','none','none',2,1,90000,4,16,''); INSERT INTO "SERVICES" VALUES(18,'yes','sysinv-inv','initial','initial','none','none',2,1,90000,4,16,'/var/run/sysinv-api.pid');