From 58c0df52016d5c860baf5aa1a379ce7b4613db06 Mon Sep 17 00:00:00 2001 From: Sean Eagan Date: Mon, 22 Apr 2019 16:28:27 -0500 Subject: [PATCH] Extract pre-update actions out of tiller handler This is a pre-requisite for Helm 3 integration, so that these actions run regardless of whether we are going through the tiller handler. Change-Id: I97d7bcc823d11b527fcdaa7967fcab62af1c8161 --- armada/exceptions/armada_exceptions.py | 15 ++ armada/exceptions/tiller_exceptions.py | 40 ---- armada/handlers/chart_deploy.py | 14 +- armada/handlers/pre_update_actions.py | 217 ++++++++++++++++++ armada/handlers/tiller.py | 190 +-------------- armada/tests/unit/handlers/test_armada.py | 2 - armada/tests/unit/handlers/test_tiller.py | 13 -- .../exceptions/armada-exceptions.inc | 5 + .../exceptions/tiller-exceptions.inc | 10 - 9 files changed, 245 insertions(+), 261 deletions(-) create mode 100644 armada/handlers/pre_update_actions.py diff --git a/armada/exceptions/armada_exceptions.py b/armada/exceptions/armada_exceptions.py index 078c3df2..05c6c57c 100644 --- a/armada/exceptions/armada_exceptions.py +++ b/armada/exceptions/armada_exceptions.py @@ -102,3 +102,18 @@ class DeploymentLikelyPendingException(ArmadaException): '(last deployment age={}s) < (chart wait timeout={}s)'.format( release, status, last_deployment_age, timeout)) super(DeploymentLikelyPendingException, self).__init__(self._message) + + +class PreUpdateJobDeleteException(ArmadaException): + ''' + Exception that occurs when a job deletion. + + **Troubleshoot:** + *Coming Soon* + ''' + + def __init__(self, name, namespace): + + message = 'Failed to delete k8s job {} in {}'.format(name, namespace) + + super(PreUpdateJobDeleteException, self).__init__(message) diff --git a/armada/exceptions/tiller_exceptions.py b/armada/exceptions/tiller_exceptions.py index 6582b828..5dd02f58 100644 --- a/armada/exceptions/tiller_exceptions.py +++ b/armada/exceptions/tiller_exceptions.py @@ -47,46 +47,6 @@ class ListChartsException(TillerException): message = 'There was an error listing the Helm chart releases.' -class PostUpdateJobDeleteException(TillerException): - '''Exception that occurs when a job deletion''' - - def __init__(self, name, namespace): - - message = 'Failed to delete k8s job {} in {}'.format(name, namespace) - - super(PostUpdateJobDeleteException, self).__init__(message) - - -class PostUpdateJobCreateException(TillerException): - ''' - Exception that occurs when a job creation fails. - - **Troubleshoot:** - *Coming Soon* - ''' - - def __init__(self, name, namespace): - - message = 'Failed to create k8s job {} in {}'.format(name, namespace) - - super(PostUpdateJobCreateException, self).__init__(message) - - -class PreUpdateJobDeleteException(TillerException): - ''' - Exception that occurs when a job deletion. - - **Troubleshoot:** - *Coming Soon* - ''' - - def __init__(self, name, namespace): - - message = 'Failed to delete k8s job {} in {}'.format(name, namespace) - - super(PreUpdateJobDeleteException, self).__init__(message) - - class ReleaseException(TillerException): ''' Exception that occurs when a release fails to install, upgrade, delete, diff --git a/armada/handlers/chart_deploy.py b/armada/handlers/chart_deploy.py index 43507912..644f5dcd 100644 --- a/armada/handlers/chart_deploy.py +++ b/armada/handlers/chart_deploy.py @@ -23,10 +23,11 @@ from armada.handlers import metrics from armada.handlers.chartbuilder import ChartBuilder from armada.handlers.release_diff import ReleaseDiff from armada.handlers.chart_delete import ChartDelete +from armada.handlers.pre_update_actions import PreUpdateActions from armada.handlers.schema import get_schema_info from armada.handlers.test import Test from armada.handlers.wait import ChartWait -from armada.exceptions import tiller_exceptions +from armada.exceptions import tiller_exceptions as tiller_ex import armada.utils.release as r LOG = logging.getLogger(__name__) @@ -87,7 +88,6 @@ class ChartDeploy(object): # Resolve action values = chart.get('values', {}) pre_actions = {} - post_actions = {} status = None if old_release: @@ -133,9 +133,8 @@ class ChartDeploy(object): if not self.disable_update_post and upgrade_post: LOG.warning( - 'Post upgrade actions are ignored by Armada ' + 'Post upgrade actions are ignored by Armada' 'and will not affect deployment.') - post_actions = upgrade_post try: old_values = yaml.safe_load(old_values_string) @@ -159,6 +158,9 @@ class ChartDeploy(object): def upgrade(): # do actual update timer = int(round(deadline - time.time())) + PreUpdateActions(self.tiller.k8s).execute( + pre_actions, release, namespace, chart, disable_hooks, + values, timer) LOG.info( "Upgrading release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, @@ -167,8 +169,6 @@ class ChartDeploy(object): new_chart, release_name, namespace, - pre_actions=pre_actions, - post_actions=post_actions, disable_hooks=disable_hooks, values=yaml.safe_dump(values), wait=native_wait_enabled, @@ -315,7 +315,7 @@ class ChartDeploy(object): def _test_chart(self, release_name, test_handler): success = test_handler.test_release_for_success() if not success: - raise tiller_exceptions.TestFailedException(release_name) + raise tiller_ex.TestFailedException(release_name) def get_diff(self, old_chart, old_values, new_chart, values): return ReleaseDiff(old_chart, old_values, new_chart, values).get_diff() diff --git a/armada/handlers/pre_update_actions.py b/armada/handlers/pre_update_actions.py new file mode 100644 index 00000000..4ca0c375 --- /dev/null +++ b/armada/handlers/pre_update_actions.py @@ -0,0 +1,217 @@ +# Copyright 2019 The Armada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_log import log as logging + +from armada import const +from armada.conf import get_current_chart +from armada.exceptions import armada_exceptions as ex +from armada.handlers import schema +from armada.utils.release import label_selectors + +LOG = logging.getLogger(__name__) + + +class PreUpdateActions(): + def __init__(self, k8s): + self.k8s = k8s + + def execute( + self, actions, release_name, namespace, chart, disable_hooks, + values, timeout): + ''' + :param actions: array of items actions + :param namespace: name of pod for actions + ''' + + # TODO: Remove when v1 doc support is removed. + try: + for action in actions.get('update', []): + name = action.get('name') + LOG.info('Updating %s ', name) + action_type = action.get('type') + labels = action.get('labels') + + self.rolling_upgrade_pod_deployment( + name, release_name, namespace, labels, action_type, chart, + disable_hooks, values, timeout) + except Exception: + LOG.exception( + "Pre-action failure: could not perform rolling upgrade for " + "%(res_type)s %(res_name)s.", { + 'res_type': action_type, + 'res_name': name + }) + raise ex.PreUpdateJobDeleteException(name, namespace) + + try: + for action in actions.get('delete', []): + name = action.get('name') + action_type = action.get('type') + labels = action.get('labels', None) + + self.delete_resources( + action_type, labels, namespace, timeout=timeout) + except Exception: + LOG.exception( + "Pre-action failure: could not delete %(res_type)s " + "%(res_name)s.", { + 'res_type': action_type, + 'res_name': name + }) + raise ex.PreUpdateJobDeleteException(name, namespace) + + def delete_resources( + self, + resource_type, + resource_labels, + namespace, + wait=False, + timeout=const.DEFAULT_TILLER_TIMEOUT): + ''' + Delete resources matching provided resource type, labels, and + namespace. + + :param resource_type: type of resource e.g. job, pod, etc. + :param resource_labels: labels for selecting the resources + :param namespace: namespace of resources + ''' + timeout = self._check_timeout(wait, timeout) + + label_selector = '' + if resource_labels is not None: + label_selector = label_selectors(resource_labels) + LOG.debug( + "Deleting resources in namespace: %s, matching " + "selectors: %s (timeout=%s).", namespace, label_selector, timeout) + + handled = False + if resource_type == 'job': + get_jobs = self.k8s.get_namespace_job( + namespace, label_selector=label_selector) + for jb in get_jobs.items: + jb_name = jb.metadata.name + + LOG.info( + "Deleting job: %s in namespace: %s", jb_name, namespace) + self.k8s.delete_job_action(jb_name, namespace, timeout=timeout) + handled = True + + # TODO: Remove when v1 doc support is removed. + chart = get_current_chart() + schema_info = schema.get_schema_info(chart['schema']) + job_implies_cronjob = schema_info.version < 2 + implied_cronjob = resource_type == 'job' and job_implies_cronjob + + if resource_type == 'cronjob' or implied_cronjob: + get_jobs = self.k8s.get_namespace_cron_job( + namespace, label_selector=label_selector) + for jb in get_jobs.items: + jb_name = jb.metadata.name + + # TODO: Remove when v1 doc support is removed. + if implied_cronjob: + LOG.warn( + "Deleting cronjobs via `type: job` is " + "deprecated, use `type: cronjob` instead") + + LOG.info( + "Deleting cronjob %s in namespace: %s", jb_name, namespace) + self.k8s.delete_cron_job_action(jb_name, namespace) + handled = True + + if resource_type == 'pod': + release_pods = self.k8s.get_namespace_pod( + namespace, label_selector=label_selector) + for pod in release_pods.items: + pod_name = pod.metadata.name + + LOG.info( + "Deleting pod %s in namespace: %s", pod_name, namespace) + self.k8s.delete_pod_action(pod_name, namespace) + if wait: + self.k8s.wait_for_pod_redeployment(pod_name, namespace) + handled = True + + if not handled: + LOG.error( + 'No resources found with labels=%s type=%s namespace=%s', + resource_labels, resource_type, namespace) + + def rolling_upgrade_pod_deployment( + self, + name, + release_name, + namespace, + resource_labels, + action_type, + chart, + disable_hooks, + values, + timeout=const.DEFAULT_TILLER_TIMEOUT): + ''' + update statefulsets (daemon, stateful) + ''' + + if action_type == 'daemonset': + + LOG.info('Updating: %s', action_type) + + label_selector = '' + + if resource_labels is not None: + label_selector = label_selectors(resource_labels) + + get_daemonset = self.k8s.get_namespace_daemon_set( + namespace, label_selector=label_selector) + + for ds in get_daemonset.items: + ds_name = ds.metadata.name + ds_labels = ds.metadata.labels + if ds_name == name: + LOG.info( + "Deleting %s : %s in %s", action_type, ds_name, + namespace) + self.k8s.delete_daemon_action(ds_name, namespace) + + # update the daemonset yaml + template = self.get_chart_templates( + ds_name, name, release_name, namespace, chart, + disable_hooks, values) + template['metadata']['labels'] = ds_labels + template['spec']['template']['metadata'][ + 'labels'] = ds_labels + + self.k8s.create_daemon_action( + namespace=namespace, template=template) + + # delete pods + self.delete_resources( + 'pod', + resource_labels, + namespace, + wait=True, + timeout=timeout) + + else: + LOG.error("Unable to exectue name: % type: %s", name, action_type) + + def _check_timeout(self, wait, timeout): + if timeout is None or timeout <= 0: + if wait: + LOG.warn( + 'Pre-update actions timeout is invalid or unspecified, ' + 'using default %ss.', self.timeout) + timeout = self.timeout + return timeout diff --git a/armada/handlers/tiller.py b/armada/handlers/tiller.py index c685bbdd..ed6077a4 100644 --- a/armada/handlers/tiller.py +++ b/armada/handlers/tiller.py @@ -29,12 +29,10 @@ from oslo_log import log as logging import yaml from armada import const -from armada.conf import get_current_chart from armada.exceptions import tiller_exceptions as ex from armada.handlers.k8s import K8s -from armada.handlers import schema from armada.utils import helm -from armada.utils.release import label_selectors, get_release_status +from armada.utils.release import get_release_status TILLER_VERSION = b'2.16.9' GRPC_EPSILON = 60 @@ -301,51 +299,6 @@ class Tiller(object): LOG.info(template_name) return template - def _pre_update_actions( - self, actions, release_name, namespace, chart, disable_hooks, - values, timeout): - ''' - :param actions: array of items actions - :param namespace: name of pod for actions - ''' - - # TODO: Remove when v1 doc support is removed. - try: - for action in actions.get('update', []): - name = action.get('name') - LOG.info('Updating %s ', name) - action_type = action.get('type') - labels = action.get('labels') - - self.rolling_upgrade_pod_deployment( - name, release_name, namespace, labels, action_type, chart, - disable_hooks, values, timeout) - except Exception: - LOG.exception( - "Pre-action failure: could not perform rolling upgrade for " - "%(res_type)s %(res_name)s.", { - 'res_type': action_type, - 'res_name': name - }) - raise ex.PreUpdateJobDeleteException(name, namespace) - - try: - for action in actions.get('delete', []): - name = action.get('name') - action_type = action.get('type') - labels = action.get('labels', None) - - self.delete_resources( - action_type, labels, namespace, timeout=timeout) - except Exception: - LOG.exception( - "Pre-action failure: could not delete %(res_type)s " - "%(res_name)s.", { - 'res_type': action_type, - 'res_name': name - }) - raise ex.PreUpdateJobDeleteException(name, namespace) - def list_charts(self): ''' List Helm Charts from Latest Releases @@ -375,8 +328,6 @@ class Tiller(object): chart, release, namespace, - pre_actions=None, - post_actions=None, disable_hooks=False, values=None, wait=False, @@ -397,10 +348,6 @@ class Tiller(object): else: values = Config(raw=values) - self._pre_update_actions( - pre_actions, release, namespace, chart, disable_hooks, values, - timeout) - update_msg = None # build release install request try: @@ -617,141 +564,6 @@ class Tiller(object): status = self.get_release_status(release) raise ex.ReleaseException(release, status, 'Delete') - def delete_resources( - self, - resource_type, - resource_labels, - namespace, - wait=False, - timeout=const.DEFAULT_TILLER_TIMEOUT): - ''' - Delete resources matching provided resource type, labels, and - namespace. - - :param resource_type: type of resource e.g. job, pod, etc. - :param resource_labels: labels for selecting the resources - :param namespace: namespace of resources - ''' - timeout = self._check_timeout(wait, timeout) - - label_selector = '' - if resource_labels is not None: - label_selector = label_selectors(resource_labels) - LOG.debug( - "Deleting resources in namespace: %s, matching " - "selectors: %s (timeout=%s).", namespace, label_selector, timeout) - - handled = False - if resource_type == 'job': - get_jobs = self.k8s.get_namespace_job( - namespace, label_selector=label_selector) - for jb in get_jobs.items: - jb_name = jb.metadata.name - - LOG.info( - "Deleting job: %s in namespace: %s", jb_name, namespace) - self.k8s.delete_job_action(jb_name, namespace, timeout=timeout) - handled = True - - # TODO: Remove when v1 doc support is removed. - chart = get_current_chart() - schema_info = schema.get_schema_info(chart['schema']) - job_implies_cronjob = schema_info.version < 2 - implied_cronjob = resource_type == 'job' and job_implies_cronjob - - if resource_type == 'cronjob' or implied_cronjob: - get_jobs = self.k8s.get_namespace_cron_job( - namespace, label_selector=label_selector) - for jb in get_jobs.items: - jb_name = jb.metadata.name - - # TODO: Remove when v1 doc support is removed. - if implied_cronjob: - LOG.warn( - "Deleting cronjobs via `type: job` is " - "deprecated, use `type: cronjob` instead") - - LOG.info( - "Deleting cronjob %s in namespace: %s", jb_name, namespace) - self.k8s.delete_cron_job_action(jb_name, namespace) - handled = True - - if resource_type == 'pod': - release_pods = self.k8s.get_namespace_pod( - namespace, label_selector=label_selector) - for pod in release_pods.items: - pod_name = pod.metadata.name - - LOG.info( - "Deleting pod %s in namespace: %s", pod_name, namespace) - self.k8s.delete_pod_action(pod_name, namespace) - if wait: - self.k8s.wait_for_pod_redeployment(pod_name, namespace) - handled = True - - if not handled: - LOG.error( - 'No resources found with labels=%s type=%s namespace=%s', - resource_labels, resource_type, namespace) - - def rolling_upgrade_pod_deployment( - self, - name, - release_name, - namespace, - resource_labels, - action_type, - chart, - disable_hooks, - values, - timeout=const.DEFAULT_TILLER_TIMEOUT): - ''' - update statefulsets (daemon, stateful) - ''' - - if action_type == 'daemonset': - - LOG.info('Updating: %s', action_type) - - label_selector = '' - - if resource_labels is not None: - label_selector = label_selectors(resource_labels) - - get_daemonset = self.k8s.get_namespace_daemon_set( - namespace, label_selector=label_selector) - - for ds in get_daemonset.items: - ds_name = ds.metadata.name - ds_labels = ds.metadata.labels - if ds_name == name: - LOG.info( - "Deleting %s : %s in %s", action_type, ds_name, - namespace) - self.k8s.delete_daemon_action(ds_name, namespace) - - # update the daemonset yaml - template = self.get_chart_templates( - ds_name, name, release_name, namespace, chart, - disable_hooks, values) - template['metadata']['labels'] = ds_labels - template['spec']['template']['metadata'][ - 'labels'] = ds_labels - - self.k8s.create_daemon_action( - namespace=namespace, template=template) - - # delete pods - self.delete_resources( - 'pod', - resource_labels, - namespace, - wait=True, - timeout=timeout) - - else: - LOG.error("Unable to exectue name: % type: %s", name, action_type) - def rollback_release( self, release_name, diff --git a/armada/tests/unit/handlers/test_armada.py b/armada/tests/unit/handlers/test_armada.py index fe523227..51369747 100644 --- a/armada/tests/unit/handlers/test_armada.py +++ b/armada/tests/unit/handlers/test_armada.py @@ -480,8 +480,6 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase): ['release_prefix'], chart['release']), chart['namespace'], - pre_actions={}, - post_actions={}, disable_hooks=disable_hooks, force=force, recreate_pods=recreate_pods, diff --git a/armada/tests/unit/handlers/test_tiller.py b/armada/tests/unit/handlers/test_tiller.py index ea6b3f22..835f8711 100644 --- a/armada/tests/unit/handlers/test_tiller.py +++ b/armada/tests/unit/handlers/test_tiller.py @@ -13,7 +13,6 @@ # limitations under the License. import mock -from mock import MagicMock from armada.exceptions import tiller_exceptions as ex from armada.handlers import tiller @@ -464,12 +463,6 @@ class TillerTestCase(base.ArmadaTestCase): tiller_obj = tiller.Tiller('host', '8080', None) - # TODO: Test these methods as well, either by unmocking, or adding - # separate tests for them. - tiller_obj._pre_update_actions = MagicMock() - - pre_actions = {} - post_actions = {} disable_hooks = False wait = True timeout = 123 @@ -480,8 +473,6 @@ class TillerTestCase(base.ArmadaTestCase): chart, release, namespace, - pre_actions=pre_actions, - post_actions=post_actions, disable_hooks=disable_hooks, values=values, wait=wait, @@ -489,10 +480,6 @@ class TillerTestCase(base.ArmadaTestCase): force=force, recreate_pods=recreate_pods) - tiller_obj._pre_update_actions.assert_called_once_with( - pre_actions, release, namespace, chart, disable_hooks, values, - timeout) - mock_update_release_request.assert_called_once_with( chart=chart, name=release, diff --git a/doc/source/operations/exceptions/armada-exceptions.inc b/doc/source/operations/exceptions/armada-exceptions.inc index 49dff73d..8f8b3e65 100644 --- a/doc/source/operations/exceptions/armada-exceptions.inc +++ b/doc/source/operations/exceptions/armada-exceptions.inc @@ -53,3 +53,8 @@ Armada Exceptions :members: :show-inheritance: :undoc-members: + +.. autoexception:: PreUpdateJobDeleteException + :members: + :show-inheritance: + :undoc-members: diff --git a/doc/source/operations/exceptions/tiller-exceptions.inc b/doc/source/operations/exceptions/tiller-exceptions.inc index a175f592..5a72f21e 100644 --- a/doc/source/operations/exceptions/tiller-exceptions.inc +++ b/doc/source/operations/exceptions/tiller-exceptions.inc @@ -29,16 +29,6 @@ Tiller Exceptions :show-inheritance: :undoc-members: -.. autoexception:: PostUpdateJobCreateException - :members: - :show-inheritance: - :undoc-members: - -.. autoexception:: PreUpdateJobDeleteException - :members: - :show-inheritance: - :undoc-members: - .. autoexception:: ReleaseException :members: :show-inheritance: