Allow Armada to clone OpenStack infra repos

-No longer attempts to ping repo
-Move all source locating (git or local) to preflight
-Only unique repos are cloned now
-Catches any source/url errors in preflight and raises exception
This commit is contained in:
Tim Heyer 2017-07-12 21:22:22 +00:00 committed by Alexis Rivera DeLa Torre
parent 62c05f569a
commit 701ac2fa8f
7 changed files with 140 additions and 110 deletions

View File

@ -22,7 +22,6 @@ def applyCharts(args):
args.disable_update_pre, args.disable_update_pre,
args.disable_update_post, args.disable_update_post,
args.enable_chart_cleanup, args.enable_chart_cleanup,
args.skip_pre_flight,
args.dry_run, args.dry_run,
args.wait, args.wait,
args.timeout, args.timeout,
@ -36,8 +35,6 @@ class ApplyChartsCommand(cmd.Command):
help='Armada yaml file') help='Armada yaml file')
parser.add_argument('--dry-run', action='store_true', parser.add_argument('--dry-run', action='store_true',
default=False, help='Run charts with dry run') default=False, help='Run charts with dry run')
parser.add_argument('--skip-pre-flight', action='store_true',
default=False, help='Skip Pre Flight')
parser.add_argument('--debug-logging', action='store_true', parser.add_argument('--debug-logging', action='store_true',
default=False, help='Show debug logs') default=False, help='Show debug logs')
parser.add_argument('--disable-update-pre', action='store_true', parser.add_argument('--disable-update-pre', action='store_true',

View File

@ -43,7 +43,6 @@ class Armada(object):
disable_update_pre=False, disable_update_pre=False,
disable_update_post=False, disable_update_post=False,
enable_chart_cleanup=False, enable_chart_cleanup=False,
skip_pre_flight=False,
dry_run=False, dry_run=False,
wait=False, wait=False,
timeout=DEFAULT_TIMEOUT, timeout=DEFAULT_TIMEOUT,
@ -55,7 +54,6 @@ class Armada(object):
self.disable_update_pre = disable_update_pre self.disable_update_pre = disable_update_pre
self.disable_update_post = disable_update_post self.disable_update_post = disable_update_post
self.enable_chart_cleanup = enable_chart_cleanup self.enable_chart_cleanup = enable_chart_cleanup
self.skip_pre_flight = skip_pre_flight
self.dry_run = dry_run self.dry_run = dry_run
self.wait = wait self.wait = wait
self.timeout = timeout self.timeout = timeout
@ -75,20 +73,55 @@ class Armada(object):
if chart_name == name: if chart_name == name:
return chart, values return chart, values
def pre_flight_checks(self): def pre_flight_ops(self):
'''
Perform a series of checks and operations to ensure proper deployment
'''
# Ensure tiller is available and yaml is valid
if not self.tiller.tiller_status():
raise Exception("Tiller Services is not Available")
if not lint.valid_manifest(self.config):
raise Exception("Invalid Armada Manifest")
# Clone the chart sources
#
# We only support a git source type right now, which can also
# handle git:// local paths as well
repos = {}
for group in self.config.get('armada').get('charts'): for group in self.config.get('armada').get('charts'):
for ch in group.get('chart_group'): for ch in group.get('chart_group'):
location = ch.get('chart').get('source').get('location') location = ch.get('chart').get('source').get('location')
ct_type = ch.get('chart').get('source').get('type') ct_type = ch.get('chart').get('source').get('type')
reference = ch.get('chart').get('source').get('reference')
subpath = ch.get('chart').get('source').get('subpath')
if ct_type == 'git' and not git.check_available_repo(location): if ct_type == 'local':
raise ValueError(str("Invalid Url Path: " + location)) ch.get('chart')['source_dir'] = (location, subpath)
elif ct_type == 'git':
if location not in repos.keys():
try:
LOG.info('Cloning repo: %s', location)
repo_dir = git.git_clone(location, reference)
except Exception as e:
raise ValueError(e)
repos[location] = repo_dir
ch.get('chart')['source_dir'] = (repo_dir, subpath)
else:
ch.get('chart')['source_dir'] = (repos.get(location),
subpath)
else:
raise Exception("Unknown source type %s for chart %s",
ct_type, ch.get('chart').get('name'))
if not self.tiller.tiller_status(): def post_flight_ops(self):
raise Exception("Tiller Services is not Available") '''
Operations to run after deployment process has terminated
if not lint.valid_manifest(self.config): '''
raise Exception("Invalid Armada Manifest") # Delete git repos cloned for deployment
for group in self.config.get('armada').get('charts'):
for ch in group.get('chart_group'):
if ch.get('chart').get('source').get('type') == 'git':
git.source_cleanup(ch.get('chart').get('source_dir')[0])
def sync(self): def sync(self):
''' '''
@ -98,11 +131,8 @@ class Armada(object):
# TODO: (gardlt) we need to break up this func into # TODO: (gardlt) we need to break up this func into
# a more cleaner format # a more cleaner format
# extract known charts on tiller right now # extract known charts on tiller right now
if not self.skip_pre_flight: LOG.info("Performing Pre-Flight Operations")
LOG.info("Performing Pre-Flight Checks") self.pre_flight_ops()
self.pre_flight_checks()
else:
LOG.info("Skipping Pre-Flight Checks")
known_releases = self.tiller.list_charts() known_releases = self.tiller.list_charts()
prefix = self.config.get('armada').get('release_prefix') prefix = self.config.get('armada').get('release_prefix')
@ -209,7 +239,8 @@ class Armada(object):
LOG.debug("Cleaning up chart source in %s", LOG.debug("Cleaning up chart source in %s",
chartbuilder.source_directory) chartbuilder.source_directory)
chartbuilder.source_cleanup() LOG.info("Performing Post-Flight Operations")
self.post_flight_ops()
if self.enable_chart_cleanup: if self.enable_chart_cleanup:
self.tiller.chart_cleanup(prefix, self.config['armada']['charts']) self.tiller.chart_cleanup(prefix, self.config['armada']['charts'])

View File

@ -21,8 +21,6 @@ from hapi.chart.metadata_pb2 import Metadata
from hapi.chart.config_pb2 import Config from hapi.chart.config_pb2 import Config
from supermutes.dot import dotify from supermutes.dot import dotify
from ..utils.git import git_clone, source_cleanup
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log as logging from oslo_log import log as logging
@ -62,48 +60,14 @@ class ChartBuilder(object):
self.chart = chart self.chart = chart
# extract, pull, whatever the chart from its source # extract, pull, whatever the chart from its source
self.source_directory = self.source_clone() self.source_directory = self.get_source_path()
def source_clone(self): def get_source_path(self):
''' '''
Clone the charts source Return the joined path of the source directory and subpath
We only support a git source type right now, which can also
handle git:// local paths as well
''' '''
return os.path.join(self.chart.source_dir[0],
if self.chart.source.type == 'git': self.chart.source_dir[1])
if self.parent:
LOG.info("Cloning %s/%s as dependency for %s",
self.chart.source.location,
self.chart.source.subpath,
self.parent)
else:
LOG.info("Cloning %s/%s for release %s",
self.chart.source.location,
self.chart.source.subpath,
self.chart.release_name)
self._source_tmp_dir = git_clone(self.chart.source.location,
self.chart.source.reference)
return os.path.join(self._source_tmp_dir,
self.chart.source.subpath)
if self.chart.source.type == 'local':
return os.path.join(self.chart.source.location,
self.chart.source.subpath)
else:
LOG.exception("Unknown source type %s for chart %s",
self.chart.name,
self.chart.source.type)
def source_cleanup(self):
'''
Cleanup source
'''
if self.chart.source.type == 'git':
source_cleanup(self._source_tmp_dir)
def get_metadata(self): def get_metadata(self):
''' '''

View File

@ -22,10 +22,10 @@ class ArmadaTestCase(unittest.TestCase):
namespace: test namespace: test
values: {} values: {}
source: source:
type: null type: git
location: null location: git://github.com/dummy/armada
subpath: null subpath: chart_1
reference: null reference: master
dependencies: [] dependencies: []
timeout: 50 timeout: 50
@ -35,22 +35,78 @@ class ArmadaTestCase(unittest.TestCase):
namespace: test namespace: test
values: {} values: {}
source: source:
type: null type: local
location: null location: /tmp/dummy/armada
subpath: null subpath: chart_2
reference: null reference: null
dependencies: [] dependencies: []
timeout: 5 timeout: 5
""" """
@mock.patch('armada.handlers.armada.git')
@mock.patch('armada.handlers.armada.lint')
@mock.patch('armada.handlers.armada.Tiller')
def test_pre_flight_ops(self, mock_tiller, mock_lint, mock_git):
'''Test pre-flight checks and operations'''
armada = Armada('')
armada.tiller = mock_tiller
armada.config = yaml.load(self.test_yaml)
CHART_SOURCES = [('git://github.com/dummy/armada', 'chart_1'),
('/tmp/dummy/armada', 'chart_2')]
# mock methods called by pre_flight_ops()
mock_tiller.tiller_status.return_value = True
mock_lint.valid_manifest.return_value = True
mock_git.git_clone.return_value = CHART_SOURCES[0][0]
armada.pre_flight_ops()
mock_git.git_clone.assert_called_once_with(CHART_SOURCES[0][0],
'master')
for group in armada.config.get('armada').get('charts'):
for counter, chart in enumerate(group.get('chart_group')):
self.assertEqual(chart.get('chart').get('source_dir')[0],
CHART_SOURCES[counter][0])
self.assertEqual(chart.get('chart').get('source_dir')[1],
CHART_SOURCES[counter][1])
@mock.patch('armada.handlers.armada.git')
@mock.patch('armada.handlers.armada.lint')
@mock.patch('armada.handlers.armada.Tiller')
def test_post_flight_ops(self, mock_tiller, mock_lint, mock_git):
'''Test post-flight operations'''
armada = Armada('')
armada.tiller = mock_tiller
armada.config = yaml.load(self.test_yaml)
CHART_SOURCES = [('git://github.com/dummy/armada', 'chart_1'),
('/tmp/dummy/armada', 'chart_2')]
# mock methods called by pre_flight_ops()
mock_tiller.tiller_status.return_value = True
mock_lint.valid_manifest.return_value = True
mock_git.git_clone.return_value = CHART_SOURCES[0][0]
armada.pre_flight_ops()
armada.post_flight_ops()
for group in yaml.load(self.test_yaml).get('armada').get('charts'):
for counter, chart in enumerate(group.get('chart_group')):
if chart.get('chart').get('source').get('type') == 'git':
mock_git.source_cleanup \
.assert_called_with(CHART_SOURCES[counter][0])
@mock.patch.object(Armada, 'post_flight_ops')
@mock.patch.object(Armada, 'pre_flight_ops')
@mock.patch('armada.handlers.armada.ChartBuilder') @mock.patch('armada.handlers.armada.ChartBuilder')
@mock.patch('armada.handlers.armada.Tiller') @mock.patch('armada.handlers.armada.Tiller')
def test_install(self, mock_tiller, mock_chartbuilder): def test_install(self, mock_tiller, mock_chartbuilder,
mock_pre_flight, mock_post_flight):
'''Test install functionality from the sync() method''' '''Test install functionality from the sync() method'''
# instantiate Armada and Tiller objects # instantiate Armada and Tiller objects
armada = Armada('', armada = Armada('',
skip_pre_flight=True,
wait=True, wait=True,
timeout=1000) timeout=1000)
armada.tiller = mock_tiller armada.tiller = mock_tiller
@ -62,9 +118,8 @@ class ArmadaTestCase(unittest.TestCase):
# mock irrelevant methods called by armada.sync() # mock irrelevant methods called by armada.sync()
mock_tiller.list_charts.return_value = [] mock_tiller.list_charts.return_value = []
mock_chartbuilder.source_clone.return_value = None mock_chartbuilder.get_source_path.return_value = None
mock_chartbuilder.get_helm_chart.return_value = None mock_chartbuilder.get_helm_chart.return_value = None
mock_chartbuilder.source_cleanup.return_value = None
armada.sync() armada.sync()

View File

@ -14,8 +14,6 @@
import unittest import unittest
import mock import mock
from supermutes.dot import dotify
# Required Oslo configuration setup # Required Oslo configuration setup
from armada.conf import default from armada.conf import default
default.register_opts() default.register_opts()
@ -74,16 +72,6 @@ class ChartBuilderTestCase(unittest.TestCase):
memory: 128Mi memory: 128Mi
""" """
def test_chartbuilder_source_clone(self):
chart = dotify(self.chart_stream)
ChartBuilder.source_clone = mock.Mock(return_value='path')
chartbuilder = ChartBuilder(chart)
resp = getattr(chartbuilder, 'source_directory', None)
self.assertIsNotNone(resp)
self.assertIsInstance(resp, basestring)
@unittest.skip("we are having wierd scenario") @unittest.skip("we are having wierd scenario")
@mock.patch('armada.handlers.chartbuilder.dotify') @mock.patch('armada.handlers.chartbuilder.dotify')
@mock.patch('armada.handlers.chartbuilder.os') @mock.patch('armada.handlers.chartbuilder.os')

View File

@ -21,7 +21,7 @@ class GitTestCase(unittest.TestCase):
@mock.patch('armada.utils.git.tempfile') @mock.patch('armada.utils.git.tempfile')
@mock.patch('armada.utils.git.pygit2') @mock.patch('armada.utils.git.pygit2')
def test_git_clone_repo_pass(self, mock_pygit, mock_temp): def test_git_clone_good_url(self, mock_pygit, mock_temp):
mock_temp.mkdtemp.return_value = '/tmp/armada' mock_temp.mkdtemp.return_value = '/tmp/armada'
mock_pygit.clone_repository.return_value = "Repository" mock_pygit.clone_repository.return_value = "Repository"
url = 'http://github.com/att-comdev/armada' url = 'http://github.com/att-comdev/armada'
@ -37,17 +37,24 @@ class GitTestCase(unittest.TestCase):
def test_git_clone_bad_url(self): def test_git_clone_bad_url(self):
url = 'http://github.com/dummy/armada' url = 'http://github.com/dummy/armada'
dir = git.git_clone(url)
self.assertFalse(dir) with self.assertRaises(Exception):
git.git_clone(url)
def test_check_available_repo_pass(self): @mock.patch('armada.utils.git.shutil')
url = 'http://github.com/att-comdev/armada' @mock.patch('armada.utils.git.path')
resp = git.check_available_repo(url) def test_source_cleanup(self, mock_path, mock_shutil):
mock_path.exists.return_value = True
path = 'armada'
self.assertTrue(resp) git.source_cleanup(path)
mock_shutil.rmtree.assert_called_with(path)
def test_check_available_repo_dummy_url(self): @mock.patch('armada.utils.git.shutil')
url = 'http://github.com/dummy/armada' @mock.patch('armada.utils.git.path')
resp = git.check_available_repo(url) def test_source_cleanup_bad_path(self, mock_path, mock_shutil):
self.assertFalse(resp) mock_path.exists.return_value = False
path = 'armada'
git.source_cleanup(path)
mock_shutil.rmtree.assert_not_called()

View File

@ -1,16 +1,14 @@
import pygit2 import pygit2
import tempfile import tempfile
import shutil import shutil
import requests from os import path
HTTP_OK = 200
def git_clone(repo_url, branch='master'): def git_clone(repo_url, branch='master'):
''' '''
clones repo to a /tmp/ dir clones repo to a /tmp/ dir
''' '''
if repo_url == '' or not check_available_repo(repo_url): if repo_url == '':
return False return False
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp') _tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
@ -22,15 +20,5 @@ def source_cleanup(target_dir):
''' '''
Clean up source Clean up source
''' '''
shutil.rmtree(target_dir) if path.exists(target_dir):
shutil.rmtree(target_dir)
def check_available_repo(repo_url):
try:
resp = requests.get(repo_url.replace('git:', 'http:'))
if resp.status_code == HTTP_OK:
return True
return False
except Exception:
return False