diff options
authorSean Eagan <>2018-11-26 15:58:39 -0600
committerSean Eagan <>2018-11-26 15:58:39 -0600
commit673b1ed4bcb5ef4f81e5ab6b9ee28c21167dbd53 (patch)
parenta64d435de8210cda9c5a5fb25885209c33801cd9 (diff)
Workaround kubernetes python client deadlock issue
The kubernetes python client has a bug [1] which results in frequent deadlocks while being cleaned up, which causes armada to hang at the end of execution. This patchset works around that issue by mocking out the associated thread pools, since they are only needed for async kubernetes api calls, which armada does not use. [1]: Change-Id: I71fbfbe355347ae2ddd02ffd26d881368320246b
Notes (review): Code-Review+1: Kaspars Skels <> Code-Review+1: Dustin Specker <> Code-Review+1: Drew Walters <> Code-Review+2: Scott Hussey <> Code-Review+2: Bryan Strassner <> Workflow+1: Bryan Strassner <> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Thu, 29 Nov 2018 22:56:16 +0000 Reviewed-on: Project: openstack/airship-armada Branch: refs/heads/master
1 files changed, 8 insertions, 0 deletions
diff --git a/armada/handlers/ b/armada/handlers/
index 0413a2e..61a635d 100644
--- a/armada/handlers/
+++ b/armada/handlers/
@@ -17,7 +17,9 @@ import re
17from kubernetes import client 17from kubernetes import client
18from kubernetes import config 18from kubernetes import config
19from kubernetes import watch 19from kubernetes import watch
20from kubernetes.client import api_client
20from import ApiException 21from import ApiException
22from unittest.mock import Mock
21from oslo_config import cfg 23from oslo_config import cfg
22from oslo_log import log as logging 24from oslo_log import log as logging
23 25
@@ -27,6 +29,12 @@ from armada.exceptions import k8s_exceptions as exceptions
27CONF = cfg.CONF 29CONF = cfg.CONF
28LOG = logging.getLogger(__name__) 30LOG = logging.getLogger(__name__)
29 31
32# TODO: Remove after this bug is fixed and we have uplifted to a fixed version:
34# Avoid creating thread pools in kubernetes api_client.
35_dummy_pool = Mock()
36api_client.ThreadPool = lambda *args, **kwargs: _dummy_pool
30 38
31class K8s(object): 39class K8s(object):
32 ''' 40 '''