From ed38cc693bee0ce70c4c88558e72a1e5c2a8f7dd Mon Sep 17 00:00:00 2001 From: Angie Wang Date: Mon, 24 Jun 2019 09:54:01 -0400 Subject: [PATCH] Fix the apply issue after uploading an app twice During app upload, Armada container is started to validate the manifest. The k8s file (/opt/platform/19.01/armada/admin.conf) on the host is mounted into the container, it will be deleted on the host if there has any exceptions during container startup. For some reasons(system is unhealthy), it took more than 20s to start up the container by the first upload action, the second upload was issued during the container startup and went to exception due to Conflict. The container was started up after the k8s config file removed on the host so it mounts a non-exist file which causes an empty dir(/opt/platform/19.01/armada/admin.conf/) created. The commit adds a lock to lock the function that starts Armada container to ensure that only one thread can start the container at a time. Change-Id: Ibd02d6ad5eac29bc79f0eee95e930eeccf4685ee Closes-Bug: 1830954 Signed-off-by: Angie Wang --- sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py index 6989d83f37..5be21859ce 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py @@ -1865,6 +1865,7 @@ class DockerHelper(object): def __init__(self, dbapi): self._dbapi = dbapi + self._lock = threading.Lock() self.k8s_registry = None self.gcr_registry = None self.quay_registry = None @@ -1967,7 +1968,13 @@ class DockerHelper(object): try: client = docker.from_env(timeout=INSTALLATION_TIMEOUT) - armada_svc = self._start_armada_service(client) + + # It causes problem if multiple threads attempt to start the + # same container, so add lock to ensure only one thread can + # start the Armada container at a time + with self._lock: + armada_svc = self._start_armada_service(client) + if armada_svc: if request == 'validate': cmd = 'armada validate ' + manifest_file