Redeploy failed charts on 'apply'

- Search through known releases during preflight and look for
 failed status
- If failed releases, purge releases before deployment

armada/handlers/armada.py

@@ -69,7 +69,7 @@ class Armada(object):
         '''
         Find a release given a list of known_releases and a release name
         '''
-        for chart_name, _, chart, values in known_releases:
+        for chart_name, _, chart, values, _ in known_releases:
             if chart_name == name:
                 return chart, values
 
@@ -83,6 +83,19 @@ class Armada(object):
         if not lint.valid_manifest(self.config):
             raise Exception("Invalid Armada Manifest")
 
+        # Purge known releases that have failed and are in the current yaml
+        prefix = self.config.get('armada').get('release_prefix')
+        failed_releases = self.get_releases_by_status('FAILED')
+        for release in failed_releases:
+            for group in self.config.get('armada').get('charts'):
+                for ch in group.get('chart_group'):
+                    ch_release_name = release_prefix(prefix, ch.get('chart')
+                                                               .get('name'))
+                    if release[0] == ch_release_name:
+                        LOG.info('Purging failed release %s '
+                                 'before deployment', release[0])
+                        self.tiller.uninstall_release(release[0])
+
         # Clone the chart sources
         #
         # We only support a git source type right now, which can also
@@ -113,15 +126,19 @@ class Armada(object):
                     raise Exception("Unknown source type %s for chart %s",
                                     ct_type, ch.get('chart').get('name'))
 
-    def post_flight_ops(self):
+    def get_releases_by_status(self, status):
         '''
-        Operations to run after deployment process has terminated
+        :params status - status string to filter releases on
+
+        Return a list of current releases with a specified status
         '''
-        # Delete git repos cloned for deployment
-        for group in self.config.get('armada').get('charts'):
-            for ch in group.get('chart_group'):
-                if ch.get('chart').get('source').get('type') == 'git':
-                    git.source_cleanup(ch.get('chart').get('source_dir')[0])
+        filtered_releases = []
+        known_releases = self.tiller.list_charts()
+        for release in known_releases:
+            if release[4] == status:
+                filtered_releases.append(release)
+
+        return filtered_releases
 
     def sync(self):
         '''
@@ -130,10 +147,10 @@ class Armada(object):
 
         # TODO: (gardlt) we need to break up this func into
         # a more cleaner format
-        # extract known charts on tiller right now
         LOG.info("Performing Pre-Flight Operations")
         self.pre_flight_ops()
 
+        # extract known charts on tiller right now
         known_releases = self.tiller.list_charts()
         prefix = self.config.get('armada').get('release_prefix')
 
@@ -245,6 +262,16 @@ class Armada(object):
         if self.enable_chart_cleanup:
             self.tiller.chart_cleanup(prefix, self.config['armada']['charts'])
 
+    def post_flight_ops(self):
+        '''
+        Operations to run after deployment process has terminated
+        '''
+        # Delete git repos cloned for deployment
+        for group in self.config.get('armada').get('charts'):
+            for ch in group.get('chart_group'):
+                if ch.get('chart').get('source').get('type') == 'git':
+                    git.source_cleanup(ch.get('chart').get('source_dir')[0])
+
     def show_diff(self, chart, installed_chart,
                   installed_values, target_chart, target_values):
         '''

armada/handlers/tiller.py

@@ -118,9 +118,13 @@ class Tiller(object):
         '''
         releases = []
         stub = ReleaseServiceStub(self.channel)
-        req = ListReleasesRequest(limit=RELEASE_LIMIT)
+        req = ListReleasesRequest(limit=RELEASE_LIMIT,
+                                  status_codes=['DEPLOYED', 'FAILED'],
+                                  sort_by='LAST_RELEASED',
+                                  sort_order='DESC')
         release_list = stub.ListReleases(req, self.timeout,
                                          metadata=self.metadata)
+
         for y in release_list:
             releases.extend(y.releases)
         return releases
@@ -129,14 +133,17 @@ class Tiller(object):
         '''
         List Helm Charts from Latest Releases
 
-        Returns list of (name, version, chart, values)
+        Returns a list of tuples in the form:
+        (name, version, chart, values, status)
         '''
         charts = []
         for latest_release in self.list_releases():
             try:
-                charts.append((latest_release.name, latest_release.version,
-                               latest_release.chart,
-                               latest_release.config.raw))
+                charts.append(
+                    (latest_release.name, latest_release.version,
+                     latest_release.chart, latest_release.config.raw,
+                     latest_release.info.status.Code.Name(
+                         latest_release.info.status.code)))
             except IndexError:
                 continue
         return charts