Allow Armada to clone OpenStack infra repos
-No longer attempts to ping repo -Move all source locating (git or local) to preflight -Only unique repos are cloned now -Catches any source/url errors in preflight and raises exception
This commit is contained in:
parent
62c05f569a
commit
701ac2fa8f
|
@ -22,7 +22,6 @@ def applyCharts(args):
|
||||||
args.disable_update_pre,
|
args.disable_update_pre,
|
||||||
args.disable_update_post,
|
args.disable_update_post,
|
||||||
args.enable_chart_cleanup,
|
args.enable_chart_cleanup,
|
||||||
args.skip_pre_flight,
|
|
||||||
args.dry_run,
|
args.dry_run,
|
||||||
args.wait,
|
args.wait,
|
||||||
args.timeout,
|
args.timeout,
|
||||||
|
@ -36,8 +35,6 @@ class ApplyChartsCommand(cmd.Command):
|
||||||
help='Armada yaml file')
|
help='Armada yaml file')
|
||||||
parser.add_argument('--dry-run', action='store_true',
|
parser.add_argument('--dry-run', action='store_true',
|
||||||
default=False, help='Run charts with dry run')
|
default=False, help='Run charts with dry run')
|
||||||
parser.add_argument('--skip-pre-flight', action='store_true',
|
|
||||||
default=False, help='Skip Pre Flight')
|
|
||||||
parser.add_argument('--debug-logging', action='store_true',
|
parser.add_argument('--debug-logging', action='store_true',
|
||||||
default=False, help='Show debug logs')
|
default=False, help='Show debug logs')
|
||||||
parser.add_argument('--disable-update-pre', action='store_true',
|
parser.add_argument('--disable-update-pre', action='store_true',
|
||||||
|
|
|
@ -43,7 +43,6 @@ class Armada(object):
|
||||||
disable_update_pre=False,
|
disable_update_pre=False,
|
||||||
disable_update_post=False,
|
disable_update_post=False,
|
||||||
enable_chart_cleanup=False,
|
enable_chart_cleanup=False,
|
||||||
skip_pre_flight=False,
|
|
||||||
dry_run=False,
|
dry_run=False,
|
||||||
wait=False,
|
wait=False,
|
||||||
timeout=DEFAULT_TIMEOUT,
|
timeout=DEFAULT_TIMEOUT,
|
||||||
|
@ -55,7 +54,6 @@ class Armada(object):
|
||||||
self.disable_update_pre = disable_update_pre
|
self.disable_update_pre = disable_update_pre
|
||||||
self.disable_update_post = disable_update_post
|
self.disable_update_post = disable_update_post
|
||||||
self.enable_chart_cleanup = enable_chart_cleanup
|
self.enable_chart_cleanup = enable_chart_cleanup
|
||||||
self.skip_pre_flight = skip_pre_flight
|
|
||||||
self.dry_run = dry_run
|
self.dry_run = dry_run
|
||||||
self.wait = wait
|
self.wait = wait
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
|
@ -75,20 +73,55 @@ class Armada(object):
|
||||||
if chart_name == name:
|
if chart_name == name:
|
||||||
return chart, values
|
return chart, values
|
||||||
|
|
||||||
def pre_flight_checks(self):
|
def pre_flight_ops(self):
|
||||||
|
'''
|
||||||
|
Perform a series of checks and operations to ensure proper deployment
|
||||||
|
'''
|
||||||
|
# Ensure tiller is available and yaml is valid
|
||||||
|
if not self.tiller.tiller_status():
|
||||||
|
raise Exception("Tiller Services is not Available")
|
||||||
|
if not lint.valid_manifest(self.config):
|
||||||
|
raise Exception("Invalid Armada Manifest")
|
||||||
|
|
||||||
|
# Clone the chart sources
|
||||||
|
#
|
||||||
|
# We only support a git source type right now, which can also
|
||||||
|
# handle git:// local paths as well
|
||||||
|
repos = {}
|
||||||
for group in self.config.get('armada').get('charts'):
|
for group in self.config.get('armada').get('charts'):
|
||||||
for ch in group.get('chart_group'):
|
for ch in group.get('chart_group'):
|
||||||
location = ch.get('chart').get('source').get('location')
|
location = ch.get('chart').get('source').get('location')
|
||||||
ct_type = ch.get('chart').get('source').get('type')
|
ct_type = ch.get('chart').get('source').get('type')
|
||||||
|
reference = ch.get('chart').get('source').get('reference')
|
||||||
|
subpath = ch.get('chart').get('source').get('subpath')
|
||||||
|
|
||||||
if ct_type == 'git' and not git.check_available_repo(location):
|
if ct_type == 'local':
|
||||||
raise ValueError(str("Invalid Url Path: " + location))
|
ch.get('chart')['source_dir'] = (location, subpath)
|
||||||
|
elif ct_type == 'git':
|
||||||
|
if location not in repos.keys():
|
||||||
|
try:
|
||||||
|
LOG.info('Cloning repo: %s', location)
|
||||||
|
repo_dir = git.git_clone(location, reference)
|
||||||
|
except Exception as e:
|
||||||
|
raise ValueError(e)
|
||||||
|
repos[location] = repo_dir
|
||||||
|
ch.get('chart')['source_dir'] = (repo_dir, subpath)
|
||||||
|
else:
|
||||||
|
ch.get('chart')['source_dir'] = (repos.get(location),
|
||||||
|
subpath)
|
||||||
|
else:
|
||||||
|
raise Exception("Unknown source type %s for chart %s",
|
||||||
|
ct_type, ch.get('chart').get('name'))
|
||||||
|
|
||||||
if not self.tiller.tiller_status():
|
def post_flight_ops(self):
|
||||||
raise Exception("Tiller Services is not Available")
|
'''
|
||||||
|
Operations to run after deployment process has terminated
|
||||||
if not lint.valid_manifest(self.config):
|
'''
|
||||||
raise Exception("Invalid Armada Manifest")
|
# Delete git repos cloned for deployment
|
||||||
|
for group in self.config.get('armada').get('charts'):
|
||||||
|
for ch in group.get('chart_group'):
|
||||||
|
if ch.get('chart').get('source').get('type') == 'git':
|
||||||
|
git.source_cleanup(ch.get('chart').get('source_dir')[0])
|
||||||
|
|
||||||
def sync(self):
|
def sync(self):
|
||||||
'''
|
'''
|
||||||
|
@ -98,11 +131,8 @@ class Armada(object):
|
||||||
# TODO: (gardlt) we need to break up this func into
|
# TODO: (gardlt) we need to break up this func into
|
||||||
# a more cleaner format
|
# a more cleaner format
|
||||||
# extract known charts on tiller right now
|
# extract known charts on tiller right now
|
||||||
if not self.skip_pre_flight:
|
LOG.info("Performing Pre-Flight Operations")
|
||||||
LOG.info("Performing Pre-Flight Checks")
|
self.pre_flight_ops()
|
||||||
self.pre_flight_checks()
|
|
||||||
else:
|
|
||||||
LOG.info("Skipping Pre-Flight Checks")
|
|
||||||
|
|
||||||
known_releases = self.tiller.list_charts()
|
known_releases = self.tiller.list_charts()
|
||||||
prefix = self.config.get('armada').get('release_prefix')
|
prefix = self.config.get('armada').get('release_prefix')
|
||||||
|
@ -209,7 +239,8 @@ class Armada(object):
|
||||||
LOG.debug("Cleaning up chart source in %s",
|
LOG.debug("Cleaning up chart source in %s",
|
||||||
chartbuilder.source_directory)
|
chartbuilder.source_directory)
|
||||||
|
|
||||||
chartbuilder.source_cleanup()
|
LOG.info("Performing Post-Flight Operations")
|
||||||
|
self.post_flight_ops()
|
||||||
|
|
||||||
if self.enable_chart_cleanup:
|
if self.enable_chart_cleanup:
|
||||||
self.tiller.chart_cleanup(prefix, self.config['armada']['charts'])
|
self.tiller.chart_cleanup(prefix, self.config['armada']['charts'])
|
||||||
|
|
|
@ -21,8 +21,6 @@ from hapi.chart.metadata_pb2 import Metadata
|
||||||
from hapi.chart.config_pb2 import Config
|
from hapi.chart.config_pb2 import Config
|
||||||
from supermutes.dot import dotify
|
from supermutes.dot import dotify
|
||||||
|
|
||||||
from ..utils.git import git_clone, source_cleanup
|
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_log import log as logging
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
@ -62,48 +60,14 @@ class ChartBuilder(object):
|
||||||
self.chart = chart
|
self.chart = chart
|
||||||
|
|
||||||
# extract, pull, whatever the chart from its source
|
# extract, pull, whatever the chart from its source
|
||||||
self.source_directory = self.source_clone()
|
self.source_directory = self.get_source_path()
|
||||||
|
|
||||||
def source_clone(self):
|
def get_source_path(self):
|
||||||
'''
|
'''
|
||||||
Clone the charts source
|
Return the joined path of the source directory and subpath
|
||||||
|
|
||||||
We only support a git source type right now, which can also
|
|
||||||
handle git:// local paths as well
|
|
||||||
'''
|
'''
|
||||||
|
return os.path.join(self.chart.source_dir[0],
|
||||||
if self.chart.source.type == 'git':
|
self.chart.source_dir[1])
|
||||||
if self.parent:
|
|
||||||
LOG.info("Cloning %s/%s as dependency for %s",
|
|
||||||
self.chart.source.location,
|
|
||||||
self.chart.source.subpath,
|
|
||||||
self.parent)
|
|
||||||
else:
|
|
||||||
LOG.info("Cloning %s/%s for release %s",
|
|
||||||
self.chart.source.location,
|
|
||||||
self.chart.source.subpath,
|
|
||||||
self.chart.release_name)
|
|
||||||
|
|
||||||
self._source_tmp_dir = git_clone(self.chart.source.location,
|
|
||||||
self.chart.source.reference)
|
|
||||||
|
|
||||||
return os.path.join(self._source_tmp_dir,
|
|
||||||
self.chart.source.subpath)
|
|
||||||
if self.chart.source.type == 'local':
|
|
||||||
return os.path.join(self.chart.source.location,
|
|
||||||
self.chart.source.subpath)
|
|
||||||
|
|
||||||
else:
|
|
||||||
LOG.exception("Unknown source type %s for chart %s",
|
|
||||||
self.chart.name,
|
|
||||||
self.chart.source.type)
|
|
||||||
|
|
||||||
def source_cleanup(self):
|
|
||||||
'''
|
|
||||||
Cleanup source
|
|
||||||
'''
|
|
||||||
if self.chart.source.type == 'git':
|
|
||||||
source_cleanup(self._source_tmp_dir)
|
|
||||||
|
|
||||||
def get_metadata(self):
|
def get_metadata(self):
|
||||||
'''
|
'''
|
||||||
|
|
|
@ -22,10 +22,10 @@ class ArmadaTestCase(unittest.TestCase):
|
||||||
namespace: test
|
namespace: test
|
||||||
values: {}
|
values: {}
|
||||||
source:
|
source:
|
||||||
type: null
|
type: git
|
||||||
location: null
|
location: git://github.com/dummy/armada
|
||||||
subpath: null
|
subpath: chart_1
|
||||||
reference: null
|
reference: master
|
||||||
dependencies: []
|
dependencies: []
|
||||||
timeout: 50
|
timeout: 50
|
||||||
|
|
||||||
|
@ -35,22 +35,78 @@ class ArmadaTestCase(unittest.TestCase):
|
||||||
namespace: test
|
namespace: test
|
||||||
values: {}
|
values: {}
|
||||||
source:
|
source:
|
||||||
type: null
|
type: local
|
||||||
location: null
|
location: /tmp/dummy/armada
|
||||||
subpath: null
|
subpath: chart_2
|
||||||
reference: null
|
reference: null
|
||||||
dependencies: []
|
dependencies: []
|
||||||
timeout: 5
|
timeout: 5
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@mock.patch('armada.handlers.armada.git')
|
||||||
|
@mock.patch('armada.handlers.armada.lint')
|
||||||
|
@mock.patch('armada.handlers.armada.Tiller')
|
||||||
|
def test_pre_flight_ops(self, mock_tiller, mock_lint, mock_git):
|
||||||
|
'''Test pre-flight checks and operations'''
|
||||||
|
armada = Armada('')
|
||||||
|
armada.tiller = mock_tiller
|
||||||
|
armada.config = yaml.load(self.test_yaml)
|
||||||
|
|
||||||
|
CHART_SOURCES = [('git://github.com/dummy/armada', 'chart_1'),
|
||||||
|
('/tmp/dummy/armada', 'chart_2')]
|
||||||
|
|
||||||
|
# mock methods called by pre_flight_ops()
|
||||||
|
mock_tiller.tiller_status.return_value = True
|
||||||
|
mock_lint.valid_manifest.return_value = True
|
||||||
|
mock_git.git_clone.return_value = CHART_SOURCES[0][0]
|
||||||
|
|
||||||
|
armada.pre_flight_ops()
|
||||||
|
|
||||||
|
mock_git.git_clone.assert_called_once_with(CHART_SOURCES[0][0],
|
||||||
|
'master')
|
||||||
|
for group in armada.config.get('armada').get('charts'):
|
||||||
|
for counter, chart in enumerate(group.get('chart_group')):
|
||||||
|
self.assertEqual(chart.get('chart').get('source_dir')[0],
|
||||||
|
CHART_SOURCES[counter][0])
|
||||||
|
self.assertEqual(chart.get('chart').get('source_dir')[1],
|
||||||
|
CHART_SOURCES[counter][1])
|
||||||
|
|
||||||
|
@mock.patch('armada.handlers.armada.git')
|
||||||
|
@mock.patch('armada.handlers.armada.lint')
|
||||||
|
@mock.patch('armada.handlers.armada.Tiller')
|
||||||
|
def test_post_flight_ops(self, mock_tiller, mock_lint, mock_git):
|
||||||
|
'''Test post-flight operations'''
|
||||||
|
armada = Armada('')
|
||||||
|
armada.tiller = mock_tiller
|
||||||
|
armada.config = yaml.load(self.test_yaml)
|
||||||
|
|
||||||
|
CHART_SOURCES = [('git://github.com/dummy/armada', 'chart_1'),
|
||||||
|
('/tmp/dummy/armada', 'chart_2')]
|
||||||
|
|
||||||
|
# mock methods called by pre_flight_ops()
|
||||||
|
mock_tiller.tiller_status.return_value = True
|
||||||
|
mock_lint.valid_manifest.return_value = True
|
||||||
|
mock_git.git_clone.return_value = CHART_SOURCES[0][0]
|
||||||
|
armada.pre_flight_ops()
|
||||||
|
|
||||||
|
armada.post_flight_ops()
|
||||||
|
|
||||||
|
for group in yaml.load(self.test_yaml).get('armada').get('charts'):
|
||||||
|
for counter, chart in enumerate(group.get('chart_group')):
|
||||||
|
if chart.get('chart').get('source').get('type') == 'git':
|
||||||
|
mock_git.source_cleanup \
|
||||||
|
.assert_called_with(CHART_SOURCES[counter][0])
|
||||||
|
|
||||||
|
@mock.patch.object(Armada, 'post_flight_ops')
|
||||||
|
@mock.patch.object(Armada, 'pre_flight_ops')
|
||||||
@mock.patch('armada.handlers.armada.ChartBuilder')
|
@mock.patch('armada.handlers.armada.ChartBuilder')
|
||||||
@mock.patch('armada.handlers.armada.Tiller')
|
@mock.patch('armada.handlers.armada.Tiller')
|
||||||
def test_install(self, mock_tiller, mock_chartbuilder):
|
def test_install(self, mock_tiller, mock_chartbuilder,
|
||||||
|
mock_pre_flight, mock_post_flight):
|
||||||
'''Test install functionality from the sync() method'''
|
'''Test install functionality from the sync() method'''
|
||||||
|
|
||||||
# instantiate Armada and Tiller objects
|
# instantiate Armada and Tiller objects
|
||||||
armada = Armada('',
|
armada = Armada('',
|
||||||
skip_pre_flight=True,
|
|
||||||
wait=True,
|
wait=True,
|
||||||
timeout=1000)
|
timeout=1000)
|
||||||
armada.tiller = mock_tiller
|
armada.tiller = mock_tiller
|
||||||
|
@ -62,9 +118,8 @@ class ArmadaTestCase(unittest.TestCase):
|
||||||
|
|
||||||
# mock irrelevant methods called by armada.sync()
|
# mock irrelevant methods called by armada.sync()
|
||||||
mock_tiller.list_charts.return_value = []
|
mock_tiller.list_charts.return_value = []
|
||||||
mock_chartbuilder.source_clone.return_value = None
|
mock_chartbuilder.get_source_path.return_value = None
|
||||||
mock_chartbuilder.get_helm_chart.return_value = None
|
mock_chartbuilder.get_helm_chart.return_value = None
|
||||||
mock_chartbuilder.source_cleanup.return_value = None
|
|
||||||
|
|
||||||
armada.sync()
|
armada.sync()
|
||||||
|
|
||||||
|
|
|
@ -14,8 +14,6 @@
|
||||||
import unittest
|
import unittest
|
||||||
import mock
|
import mock
|
||||||
|
|
||||||
from supermutes.dot import dotify
|
|
||||||
|
|
||||||
# Required Oslo configuration setup
|
# Required Oslo configuration setup
|
||||||
from armada.conf import default
|
from armada.conf import default
|
||||||
default.register_opts()
|
default.register_opts()
|
||||||
|
@ -74,16 +72,6 @@ class ChartBuilderTestCase(unittest.TestCase):
|
||||||
memory: 128Mi
|
memory: 128Mi
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def test_chartbuilder_source_clone(self):
|
|
||||||
|
|
||||||
chart = dotify(self.chart_stream)
|
|
||||||
ChartBuilder.source_clone = mock.Mock(return_value='path')
|
|
||||||
chartbuilder = ChartBuilder(chart)
|
|
||||||
resp = getattr(chartbuilder, 'source_directory', None)
|
|
||||||
|
|
||||||
self.assertIsNotNone(resp)
|
|
||||||
self.assertIsInstance(resp, basestring)
|
|
||||||
|
|
||||||
@unittest.skip("we are having wierd scenario")
|
@unittest.skip("we are having wierd scenario")
|
||||||
@mock.patch('armada.handlers.chartbuilder.dotify')
|
@mock.patch('armada.handlers.chartbuilder.dotify')
|
||||||
@mock.patch('armada.handlers.chartbuilder.os')
|
@mock.patch('armada.handlers.chartbuilder.os')
|
||||||
|
|
|
@ -21,7 +21,7 @@ class GitTestCase(unittest.TestCase):
|
||||||
|
|
||||||
@mock.patch('armada.utils.git.tempfile')
|
@mock.patch('armada.utils.git.tempfile')
|
||||||
@mock.patch('armada.utils.git.pygit2')
|
@mock.patch('armada.utils.git.pygit2')
|
||||||
def test_git_clone_repo_pass(self, mock_pygit, mock_temp):
|
def test_git_clone_good_url(self, mock_pygit, mock_temp):
|
||||||
mock_temp.mkdtemp.return_value = '/tmp/armada'
|
mock_temp.mkdtemp.return_value = '/tmp/armada'
|
||||||
mock_pygit.clone_repository.return_value = "Repository"
|
mock_pygit.clone_repository.return_value = "Repository"
|
||||||
url = 'http://github.com/att-comdev/armada'
|
url = 'http://github.com/att-comdev/armada'
|
||||||
|
@ -37,17 +37,24 @@ class GitTestCase(unittest.TestCase):
|
||||||
|
|
||||||
def test_git_clone_bad_url(self):
|
def test_git_clone_bad_url(self):
|
||||||
url = 'http://github.com/dummy/armada'
|
url = 'http://github.com/dummy/armada'
|
||||||
dir = git.git_clone(url)
|
|
||||||
|
|
||||||
self.assertFalse(dir)
|
with self.assertRaises(Exception):
|
||||||
|
git.git_clone(url)
|
||||||
|
|
||||||
def test_check_available_repo_pass(self):
|
@mock.patch('armada.utils.git.shutil')
|
||||||
url = 'http://github.com/att-comdev/armada'
|
@mock.patch('armada.utils.git.path')
|
||||||
resp = git.check_available_repo(url)
|
def test_source_cleanup(self, mock_path, mock_shutil):
|
||||||
|
mock_path.exists.return_value = True
|
||||||
|
path = 'armada'
|
||||||
|
|
||||||
self.assertTrue(resp)
|
git.source_cleanup(path)
|
||||||
|
mock_shutil.rmtree.assert_called_with(path)
|
||||||
|
|
||||||
def test_check_available_repo_dummy_url(self):
|
@mock.patch('armada.utils.git.shutil')
|
||||||
url = 'http://github.com/dummy/armada'
|
@mock.patch('armada.utils.git.path')
|
||||||
resp = git.check_available_repo(url)
|
def test_source_cleanup_bad_path(self, mock_path, mock_shutil):
|
||||||
self.assertFalse(resp)
|
mock_path.exists.return_value = False
|
||||||
|
path = 'armada'
|
||||||
|
|
||||||
|
git.source_cleanup(path)
|
||||||
|
mock_shutil.rmtree.assert_not_called()
|
||||||
|
|
|
@ -1,16 +1,14 @@
|
||||||
import pygit2
|
import pygit2
|
||||||
import tempfile
|
import tempfile
|
||||||
import shutil
|
import shutil
|
||||||
import requests
|
from os import path
|
||||||
|
|
||||||
HTTP_OK = 200
|
|
||||||
|
|
||||||
def git_clone(repo_url, branch='master'):
|
def git_clone(repo_url, branch='master'):
|
||||||
'''
|
'''
|
||||||
clones repo to a /tmp/ dir
|
clones repo to a /tmp/ dir
|
||||||
'''
|
'''
|
||||||
|
|
||||||
if repo_url == '' or not check_available_repo(repo_url):
|
if repo_url == '':
|
||||||
return False
|
return False
|
||||||
|
|
||||||
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
|
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
|
||||||
|
@ -22,15 +20,5 @@ def source_cleanup(target_dir):
|
||||||
'''
|
'''
|
||||||
Clean up source
|
Clean up source
|
||||||
'''
|
'''
|
||||||
|
if path.exists(target_dir):
|
||||||
shutil.rmtree(target_dir)
|
shutil.rmtree(target_dir)
|
||||||
|
|
||||||
|
|
||||||
def check_available_repo(repo_url):
|
|
||||||
try:
|
|
||||||
resp = requests.get(repo_url.replace('git:', 'http:'))
|
|
||||||
if resp.status_code == HTTP_OK:
|
|
||||||
return True
|
|
||||||
|
|
||||||
return False
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
Loading…
Reference in New Issue