From 673b1ed4bcb5ef4f81e5ab6b9ee28c21167dbd53 Mon Sep 17 00:00:00 2001 From: Sean Eagan Date: Mon, 26 Nov 2018 15:58:39 -0600 Subject: [PATCH] Workaround kubernetes python client deadlock issue The kubernetes python client has a bug [1] which results in frequent deadlocks while being cleaned up, which causes armada to hang at the end of execution. This patchset works around that issue by mocking out the associated thread pools, since they are only needed for async kubernetes api calls, which armada does not use. [1]: https://github.com/kubernetes-client/python/issues/411 Change-Id: I71fbfbe355347ae2ddd02ffd26d881368320246b --- armada/handlers/k8s.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/armada/handlers/k8s.py b/armada/handlers/k8s.py index 0413a2e3..61a635da 100644 --- a/armada/handlers/k8s.py +++ b/armada/handlers/k8s.py @@ -17,7 +17,9 @@ import re from kubernetes import client from kubernetes import config from kubernetes import watch +from kubernetes.client import api_client from kubernetes.client.rest import ApiException +from unittest.mock import Mock from oslo_config import cfg from oslo_log import log as logging @@ -27,6 +29,12 @@ from armada.exceptions import k8s_exceptions as exceptions CONF = cfg.CONF LOG = logging.getLogger(__name__) +# TODO: Remove after this bug is fixed and we have uplifted to a fixed version: +# https://github.com/kubernetes-client/python/issues/411 +# Avoid creating thread pools in kubernetes api_client. +_dummy_pool = Mock() +api_client.ThreadPool = lambda *args, **kwargs: _dummy_pool + class K8s(object): '''