diff --git a/armada/handlers/k8s.py b/armada/handlers/k8s.py index e0aee760..6a1045e0 100644 --- a/armada/handlers/k8s.py +++ b/armada/handlers/k8s.py @@ -46,29 +46,62 @@ class K8s(object): self.client = client.CoreV1Api() self.batch_api = client.BatchV1Api() + self.batch_v1beta1_api = client.BatchV1beta1Api() self.extension_api = client.ExtensionsV1beta1Api() def delete_job_action(self, name, namespace="default", propagation_policy='Foreground', timeout=DEFAULT_K8S_TIMEOUT): ''' - :params name - name of the job - :params namespace - name of pod that job + :params name - name of job + :params namespace - namespace of job :params propagation_policy - The Kubernetes propagation_policy to apply - to the delete. Default 'Foreground' means - that child Pods to the Job will be deleted - before the Job is marked as deleted. + to the delete. Default 'Foreground' means that child pods to the + job will be deleted before the job is marked as deleted. ''' + self._delete_job_action( + self.batch_api.list_namespaced_job, + self.batch_api.delete_namespaced_job, + "job", + name, + namespace, + propagation_policy, + timeout) + + def delete_cron_job_action(self, name, namespace="default", + propagation_policy='Foreground', + timeout=DEFAULT_K8S_TIMEOUT): + ''' + :params name - name of cron job + :params namespace - namespace of cron job + :params propagation_policy - The Kubernetes propagation_policy to apply + to the delete. Default 'Foreground' means that child pods of the + cron job will be deleted before the cron job is marked as deleted. + ''' + self._delete_job_action( + self.batch_v1beta1_api.list_namespaced_cron_job, + self.batch_v1beta1_api.delete_namespaced_cron_job, + "cron job", + name, + namespace, + propagation_policy, + timeout) + + def _delete_job_action(self, list_func, delete_func, job_type_description, + name, namespace="default", + propagation_policy='Foreground', + timeout=DEFAULT_K8S_TIMEOUT): try: - LOG.debug('Deleting job %s, Wait timeout=%s', name, timeout) + LOG.debug('Deleting %s %s, Wait timeout=%s', + job_type_description, name, timeout) body = client.V1DeleteOptions() w = watch.Watch() issue_delete = True - for event in w.stream(self.batch_api.list_namespaced_job, + for event in w.stream(list_func, namespace=namespace, timeout_seconds=timeout): if issue_delete: - self.batch_api.delete_namespaced_job( + delete_func( name=name, namespace=namespace, body=body, propagation_policy=propagation_policy) issue_delete = False @@ -77,33 +110,49 @@ class K8s(object): job_name = event['object'].metadata.name if event_type == 'DELETED' and job_name == name: - LOG.debug('Successfully deleted job %s', job_name) + LOG.debug('Successfully deleted %s %s', + job_type_description, job_name) return - err_msg = ('Reached timeout while waiting to delete job: ' - 'name=%s, namespace=%s' % (name, namespace)) + err_msg = ('Reached timeout while waiting to delete %s: ' + 'name=%s, namespace=%s' % + (job_type_description, name, namespace)) LOG.error(err_msg) raise exceptions.KubernetesWatchTimeoutException(err_msg) except ApiException as e: LOG.exception( - "Exception when deleting job: name=%s, namespace=%s", - name, namespace) + "Exception when deleting %s: name=%s, namespace=%s", + job_type_description, name, namespace) raise e def get_namespace_job(self, namespace="default", label_selector=''): ''' - :params lables - of the job - :params namespace - name of jobs + :params label_selector - labels of the jobs + :params namespace - namespace of the jobs ''' try: return self.batch_api.list_namespaced_job( namespace, label_selector=label_selector) except ApiException as e: - LOG.error("Exception getting a job: namespace=%s, label=%s: %s", + LOG.error("Exception getting jobs: namespace=%s, label=%s: %s", namespace, label_selector, e) + def get_namespace_cron_job(self, namespace="default", label_selector=''): + ''' + :params label_selector - labels of the cron jobs + :params namespace - namespace of the cron jobs + ''' + + try: + return self.batch_v1beta1_api.list_namespaced_cron_job( + namespace, label_selector=label_selector) + except ApiException as e: + LOG.error( + "Exception getting cron jobs: namespace=%s, label=%s: %s", + namespace, label_selector, e) + def create_job_action(self, name, namespace="default"): ''' :params name - name of the job diff --git a/armada/handlers/tiller.py b/armada/handlers/tiller.py index 06bd4af8..873b2889 100644 --- a/armada/handlers/tiller.py +++ b/armada/handlers/tiller.py @@ -584,15 +584,32 @@ class Tiller(object): LOG.debug("Deleting resources in namespace %s matching " "selectors %s.", namespace, label_selector) - if 'job' in resource_type: + handled = False + if resource_type == 'job': get_jobs = self.k8s.get_namespace_job(namespace, label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name LOG.info("Deleting job %s in namespace: %s", jb_name, namespace) self.k8s.delete_job_action(jb_name, namespace, timeout=timeout) + handled = True - elif 'pod' in resource_type: + if resource_type == 'cronjob' or resource_type == 'job': + get_jobs = self.k8s.get_namespace_cron_job( + namespace, label_selector) + for jb in get_jobs.items: + jb_name = jb.metadata.name + LOG.info("Deleting cron job %s in namespace: %s", + jb_name, namespace) + if resource_type == 'job': + # TODO: Eventually disallow this, allowing initially since + # some existing clients were expecting this behavior. + LOG.warning("Deleting cron jobs via `type: job` is " + "deprecated, use `type: cronjob` instead") + self.k8s.delete_cron_job_action(jb_name, namespace) + handled = True + + if resource_type == 'pod': release_pods = self.k8s.get_namespace_pod( namespace, label_selector) @@ -603,7 +620,9 @@ class Tiller(object): self.k8s.delete_namespace_pod(pod_name, namespace) if wait: self.k8s.wait_for_pod_redeployment(pod_name, namespace) - else: + handled = True + + if not handled: LOG.error("Unable to execute name: %s type: %s ", resource_name, resource_type) diff --git a/docs/source/operations/guide-build-armada-yaml.rst b/docs/source/operations/guide-build-armada-yaml.rst index e86d9a4c..990adf8c 100644 --- a/docs/source/operations/guide-build-armada-yaml.rst +++ b/docs/source/operations/guide-build-armada-yaml.rst @@ -158,7 +158,7 @@ Upgrade - Actions - Update/Delete .. note:: - Delete Actions only support type: 'job' + Delete Actions support type: 'pod', 'job', 'cronjob' Chart Example ^^^^^^^^^^^^^