summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSean Eagan <sean.eagan@att.com>2018-11-26 15:58:39 -0600
committerSean Eagan <sean.eagan@att.com>2018-11-26 15:58:39 -0600
commit673b1ed4bcb5ef4f81e5ab6b9ee28c21167dbd53 (patch)
tree200b82cf710d2deb96ddd98012dc4e90cca9118f
parenta64d435de8210cda9c5a5fb25885209c33801cd9 (diff)
Workaround kubernetes python client deadlock issue
The kubernetes python client has a bug [1] which results in frequent deadlocks while being cleaned up, which causes armada to hang at the end of execution. This patchset works around that issue by mocking out the associated thread pools, since they are only needed for async kubernetes api calls, which armada does not use. [1]: https://github.com/kubernetes-client/python/issues/411 Change-Id: I71fbfbe355347ae2ddd02ffd26d881368320246b
Notes
Notes (review): Code-Review+1: Kaspars Skels <kaspars.skels@gmail.com> Code-Review+1: Dustin Specker <dustin.specker@att.com> Code-Review+1: Drew Walters <drewwalters96@gmail.com> Code-Review+2: Scott Hussey <sthussey@att.com> Code-Review+2: Bryan Strassner <bryan.strassner@gmail.com> Workflow+1: Bryan Strassner <bryan.strassner@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Thu, 29 Nov 2018 22:56:16 +0000 Reviewed-on: https://review.openstack.org/620169 Project: openstack/airship-armada Branch: refs/heads/master
-rw-r--r--armada/handlers/k8s.py8
1 files changed, 8 insertions, 0 deletions
diff --git a/armada/handlers/k8s.py b/armada/handlers/k8s.py
index 0413a2e..61a635d 100644
--- a/armada/handlers/k8s.py
+++ b/armada/handlers/k8s.py
@@ -17,7 +17,9 @@ import re
17from kubernetes import client 17from kubernetes import client
18from kubernetes import config 18from kubernetes import config
19from kubernetes import watch 19from kubernetes import watch
20from kubernetes.client import api_client
20from kubernetes.client.rest import ApiException 21from kubernetes.client.rest import ApiException
22from unittest.mock import Mock
21from oslo_config import cfg 23from oslo_config import cfg
22from oslo_log import log as logging 24from oslo_log import log as logging
23 25
@@ -27,6 +29,12 @@ from armada.exceptions import k8s_exceptions as exceptions
27CONF = cfg.CONF 29CONF = cfg.CONF
28LOG = logging.getLogger(__name__) 30LOG = logging.getLogger(__name__)
29 31
32# TODO: Remove after this bug is fixed and we have uplifted to a fixed version:
33# https://github.com/kubernetes-client/python/issues/411
34# Avoid creating thread pools in kubernetes api_client.
35_dummy_pool = Mock()
36api_client.ThreadPool = lambda *args, **kwargs: _dummy_pool
37
30 38
31class K8s(object): 39class K8s(object):
32 ''' 40 '''