From 1e5d781fe932b18b9a44d4508e149a2a1650cfde Mon Sep 17 00:00:00 2001 From: Thiago Brito Date: Mon, 22 Feb 2021 18:10:15 -0300 Subject: [PATCH] Fixing Armada waits for Evicted pods undefinetly In the occasion of a pod being evicted due to low resource availability, armada keeps waiting for the Evicted pod to be ready. This commit removes that behavior since Kubernetes will spun up a new pod. Story: 2008645 Task: 41906 Signed-off-by: Thiago Brito Change-Id: I7263eebe357b0952375d538555536dc9f7cceff4 --- armada/handlers/wait.py | 3 +++ armada/tests/unit/handlers/test_wait.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/armada/handlers/wait.py b/armada/handlers/wait.py index e2c156f7..9c1d44a1 100644 --- a/armada/handlers/wait.py +++ b/armada/handlers/wait.py @@ -486,6 +486,9 @@ class PodWait(ResourceWait): if is_test_pod(pod): return 'helm test pod' + if pod.status.phase == 'Evicted': + return "pod was evicted" + schema_info = get_schema_info(self.chart_wait.chart['schema']) # TODO: Remove when v1 doc support is removed. if schema_info.version < 2: diff --git a/armada/tests/unit/handlers/test_wait.py b/armada/tests/unit/handlers/test_wait.py index 921facee..81b38b0d 100644 --- a/armada/tests/unit/handlers/test_wait.py +++ b/armada/tests/unit/handlers/test_wait.py @@ -218,7 +218,7 @@ class PodWaitTestCase(base.ArmadaTestCase): 'helm.sh/hook': 'test-success' }), mock_resource({'helm.sh/hook': 'test-failure'}), - mock_resource({'helm.sh/hook': 'test-success,pre-install'}) + mock_resource({'helm.sh/hook': 'test-success,pre-install'}), ] job_pods = [ mock_resource(owner_references=[mock.Mock(kind='Job')]), @@ -233,7 +233,12 @@ class PodWaitTestCase(base.ArmadaTestCase): mock_resource(owner_references=[]), mock_resource({'helm.sh/hook': 'pre-install'}), mock_resource({'key': 'value'}), - mock_resource(owner_references=[mock.Mock(kind='NotAJob')]) + mock_resource(owner_references=[mock.Mock(kind='NotAJob')]), + ] + evicted_pods = [ + mock.Mock( + metadata=mock.Mock(annotations={}, owner_references=None), + status=mock.Mock(phase='Evicted')), ] unit = self.get_unit({}, version=1) @@ -250,6 +255,10 @@ class PodWaitTestCase(base.ArmadaTestCase): for pod in included_pods: self.assertTrue(unit.include_resource(pod)) + # Validate evicted pods are excluded + for pod in evicted_pods: + self.assertFalse(unit.include_resource(pod)) + class JobWaitTestCase(base.ArmadaTestCase): def get_unit(self, labels):