diff --git a/armada/exceptions/armada_exceptions.py b/armada/exceptions/armada_exceptions.py index 80e39df0..7591d548 100644 --- a/armada/exceptions/armada_exceptions.py +++ b/armada/exceptions/armada_exceptions.py @@ -23,15 +23,3 @@ class KnownReleasesException(ArmadaException): '''Exception that occurs when no known releases are found''' message = 'No known releases found' - -class ChartSourceException(ArmadaException): - '''Exception for unknown chart source type.''' - - def __init__(self, chart_name, source_type): - self._chart_name = chart_name - self._source_type = source_type - - self._message = 'Unknown source type \"' + self._source_type + '\" for \ - chart \"' + self._chart_name + '\"' - - super(ChartSourceException, self).__init__(self._message) diff --git a/armada/exceptions/chartbuilder_exceptions.py b/armada/exceptions/chartbuilder_exceptions.py index 9554dd85..8ff27a40 100644 --- a/armada/exceptions/chartbuilder_exceptions.py +++ b/armada/exceptions/chartbuilder_exceptions.py @@ -23,7 +23,7 @@ class DependencyException(ChartBuilderException): '''Exception that occurs when dependencies cannot be resolved.''' def __init__(self, chart_name): - self._chart_name + self._chart_name = chart_name self._message = 'Failed to resolve dependencies for ' + \ self._chart_name + '.' @@ -33,7 +33,7 @@ class HelmChartBuildException(ChartBuilderException): '''Exception that occurs when Helm Chart fails to build.''' def __init__(self, chart_name): - self._chart_name + self._chart_name = chart_name self._message = 'Failed to build Helm chart for ' + \ self._chart_name + '.' diff --git a/armada/exceptions/git_exceptions.py b/armada/exceptions/git_exceptions.py deleted file mode 100644 index 142da3d5..00000000 --- a/armada/exceptions/git_exceptions.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2017 The Armada Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base_exception - -class GitException(base_exception.ArmadaBaseException): - '''Base class for Git exceptions and error handling.''' - - message = 'An unknown error occured while cloning a Git repository.' - -class GitLocationException(GitException): - '''Exception that occurs when an error occurs cloning a Git repository.''' - - def __init__(self, location): - self._location = location - self._message = self._location + ' is not a valid git repository.' - - super(GitLocationException, self).__init__(self._message) - -class SourceCleanupException(GitException): - '''Exception that occurs for an invalid dir.''' - - def __init__(self, target_dir): - self._target_dir = target_dir - self._message = self._target_dir + ' is not a valid directory.' - - super(SourceCleanupException, self).__init__(self._message) diff --git a/armada/exceptions/source_exceptions.py b/armada/exceptions/source_exceptions.py new file mode 100644 index 00000000..2fbc7f08 --- /dev/null +++ b/armada/exceptions/source_exceptions.py @@ -0,0 +1,80 @@ +# Copyright 2017 The Armada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base_exception + +class SourceException(base_exception.ArmadaBaseException): + '''Base class for Git exceptions and error handling.''' + + message = 'An unknown error occured while accessing a chart source' + +class GitLocationException(SourceException): + '''Exception that occurs when an error occurs cloning a Git repository.''' + + def __init__(self, location): + self._location = location + self._message = self._location + ' is not a valid git repository.' + + super(GitLocationException, self).__init__(self._message) + +class SourceCleanupException(SourceException): + '''Exception that occurs for an invalid dir.''' + + def __init__(self, target_dir): + self._target_dir = target_dir + self._message = self._target_dir + ' is not a valid directory.' + + super(SourceCleanupException, self).__init__(self._message) + +class TarballDownloadException(SourceException): + '''Exception that occurs when the tarball cannot be downloaded + from the provided URL + ''' + + def __init__(self, tarball_url): + self._tarball_url = tarball_url + self._message = 'Unable to download from ' + self._tarball_url + + super(TarballDownloadException, self).__init__(self._message) + +class TarballExtractException(SourceException): + '''Exception that occurs when extracting the tarball fails''' + + def __init__(self, tarball_dir): + self._tarball_dir = tarball_dir + self._message = 'Unable to extract ' + self._tarball_dir + + super(TarballExtractException, self).__init__(self._message) + +class InvalidPathException(SourceException): + '''Exception that occurs when a nonexistant path is accessed''' + + def __init__(self, path): + self._path = path + self._message = 'Unable to access path ' + self._path + + super(InvalidPathException, self).__init__(self._message) + + +class ChartSourceException(SourceException): + '''Exception for unknown chart source type.''' + + def __init__(self, chart_name, source_type): + self._chart_name = chart_name + self._source_type = source_type + + self._message = 'Unknown source type \"' + self._source_type + '\" for \ + chart \"' + self._chart_name + '\"' + + super(ChartSourceException, self).__init__(self._message) diff --git a/armada/handlers/armada.py b/armada/handlers/armada.py index 36d5cb80..b403f93a 100644 --- a/armada/handlers/armada.py +++ b/armada/handlers/armada.py @@ -24,12 +24,12 @@ from tiller import Tiller from manifest import Manifest from ..exceptions import armada_exceptions -from ..exceptions import git_exceptions +from ..exceptions import source_exceptions from ..exceptions import lint_exceptions from ..exceptions import tiller_exceptions from ..utils.release import release_prefix -from ..utils import git +from ..utils import source from ..utils import lint from ..const import KEYWORD_ARMADA, KEYWORD_GROUPS, KEYWORD_CHARTS,\ KEYWORD_PREFIX, STATUS_FAILED @@ -142,20 +142,24 @@ class Armada(object): if ct_type == 'local': ch.get('chart')['source_dir'] = (location, subpath) + elif ct_type == 'tar': + LOG.info('Downloading tarball from: %s', location) + tarball_dir = source.get_tarball(location) + ch.get('chart')['source_dir'] = (tarball_dir, subpath) elif ct_type == 'git': if location not in repos.keys(): try: LOG.info('Cloning repo: %s', location) - repo_dir = git.git_clone(location, reference) + repo_dir = source.git_clone(location, reference) except Exception: - raise git_exceptions.GitLocationException(location) + raise source_exceptions.GitLocationException(location) repos[location] = repo_dir ch.get('chart')['source_dir'] = (repo_dir, subpath) else: ch.get('chart')['source_dir'] = (repos.get(location), subpath) else: chart_name = ch.get('chart').get('chart_name') - raise armada_exceptions.ChartSourceException(ct_type, chart_name) + raise source_exceptions.ChartSourceException(ct_type, chart_name) def get_releases_by_status(self, status): ''' @@ -305,7 +309,7 @@ class Armada(object): for group in self.config.get(KEYWORD_ARMADA).get(KEYWORD_GROUPS): for ch in group.get(KEYWORD_CHARTS): if ch.get('chart').get('source').get('type') == 'git': - git.source_cleanup(ch.get('chart').get('source_dir')[0]) + source.source_cleanup(ch.get('chart').get('source_dir')[0]) def show_diff(self, chart, installed_chart, installed_values, target_chart, target_values): diff --git a/armada/handlers/chartbuilder.py b/armada/handlers/chartbuilder.py index 39979afc..c7758fe0 100644 --- a/armada/handlers/chartbuilder.py +++ b/armada/handlers/chartbuilder.py @@ -202,7 +202,7 @@ class ChartBuilder(object): templates=self.get_templates(), dependencies=dependencies, values=self.get_values(), - files=self.get_files(), ) + files=self.get_files()) except Exception: chart_name = self.chart.chart_name raise chartbuilder_exceptions.HelmChartBuildException(chart_name) diff --git a/armada/tests/unit/utils/test_git.py b/armada/tests/unit/utils/test_git.py deleted file mode 100644 index 68caff11..00000000 --- a/armada/tests/unit/utils/test_git.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2017 The Armada Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock -import unittest - -from armada.exceptions import git_exceptions - -from armada.utils import git - -class GitTestCase(unittest.TestCase): - - @mock.patch('armada.utils.git.tempfile') - @mock.patch('armada.utils.git.pygit2') - def test_git_clone_good_url(self, mock_pygit, mock_temp): - mock_temp.mkdtemp.return_value = '/tmp/armada' - mock_pygit.clone_repository.return_value = "Repository" - url = 'http://github.com/att-comdev/armada' - dir = git.git_clone(url) - - self.assertIsNotNone(dir) - - def test_git_clone_empty_url(self): - url = '' - - with self.assertRaises(Exception): - self.assertFalse(git.git_clone(url)) - - def test_git_clone_bad_url(self): - url = 'http://github.com/dummy/armada' - - with self.assertRaises(Exception): - git.git_clone(url) - - @mock.patch('armada.utils.git.shutil') - @mock.patch('armada.utils.git.path') - def test_source_cleanup(self, mock_path, mock_shutil): - mock_path.exists.return_value = True - path = 'armada' - - try: - git.source_cleanup(path) - except git_exceptions.SourceCleanupException: - pass - - mock_shutil.rmtree.assert_called_with(path) - - @unittest.skip('not handled correctly') - @mock.patch('armada.utils.git.shutil') - @mock.patch('armada.utils.git.path') - def test_source_cleanup_bad_path(self, mock_path, mock_shutil): - mock_path.exists.return_value = False - path = 'armada' - with self.assertRaises(Exception): - git.source_cleanup(path) - mock_shutil.rmtree.assert_not_called() diff --git a/armada/tests/unit/utils/test_source.py b/armada/tests/unit/utils/test_source.py new file mode 100644 index 00000000..561049e7 --- /dev/null +++ b/armada/tests/unit/utils/test_source.py @@ -0,0 +1,117 @@ +# Copyright 2017 The Armada Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unittest + +from armada.exceptions import source_exceptions + +from armada.utils import source + +class GitTestCase(unittest.TestCase): + + SOURCE_UTILS_LOCATION = 'armada.utils.source' + + @mock.patch('armada.utils.source.tempfile') + @mock.patch('armada.utils.source.pygit2') + def test_git_clone_good_url(self, mock_pygit, mock_temp): + mock_temp.mkdtemp.return_value = '/tmp/armada' + mock_pygit.clone_repository.return_value = "Repository" + url = 'http://github.com/att-comdev/armada' + dir = source.git_clone(url) + + self.assertIsNotNone(dir) + + def test_git_clone_empty_url(self): + url = '' + + with self.assertRaises(Exception): + self.assertFalse(source.git_clone(url)) + + def test_git_clone_bad_url(self): + url = 'http://github.com/dummy/armada' + + with self.assertRaises(Exception): + source.git_clone(url) + + @mock.patch('armada.utils.source.tempfile') + @mock.patch('armada.utils.source.requests') + def test_tarball_download(self, mock_requests, mock_temp): + url = 'http://localhost:8879/charts/mariadb-0.1.0.tgz' + mock_temp.mkstemp.return_value = (None, '/tmp/armada') + mock_response = mock.Mock() + mock_response.content = 'some string' + mock_requests.get.return_value = mock_response + + mock_open = mock.mock_open() + with mock.patch('{}.open'.format(self.SOURCE_UTILS_LOCATION), + mock_open, create=True): + source.download_tarball(url) + + mock_temp.mkstemp.assert_called_once() + mock_requests.get.assert_called_once_with(url) + mock_open.assert_called_once_with('/tmp/armada', 'wb') + mock_open().write.assert_called_once_with( + mock_requests.get(url).content) + + @mock.patch('armada.utils.source.tempfile') + @mock.patch('armada.utils.source.path') + @mock.patch('armada.utils.source.tarfile') + def test_tarball_extract(self, mock_tarfile, mock_path, mock_temp): + mock_path.exists.return_value = True + mock_temp.mkdtemp.return_value = '/tmp/armada' + mock_opened_file = mock.Mock() + mock_tarfile.open.return_value = mock_opened_file + + path = '/tmp/mariadb-0.1.0.tgz' + source.extract_tarball(path) + + mock_path.exists.assert_called_once() + mock_temp.mkdtemp.assert_called_once() + mock_tarfile.open.assert_called_once_with(path) + mock_opened_file.extractall.assert_called_once_with('/tmp/armada') + + @mock.patch('armada.utils.source.path') + @mock.patch('armada.utils.source.tarfile') + def test_tarball_extract_bad_path(self, mock_tarfile, mock_path): + mock_path.exists.return_value = False + path = '/tmp/armada' + with self.assertRaises(Exception): + source.extract_tarball(path) + + mock_tarfile.open.assert_not_called() + mock_tarfile.extractall.assert_not_called() + + @mock.patch('armada.utils.source.shutil') + @mock.patch('armada.utils.source.path') + def test_source_cleanup(self, mock_path, mock_shutil): + mock_path.exists.return_value = True + path = 'armada' + + try: + source.source_cleanup(path) + except source_exceptions.SourceCleanupException: + pass + + mock_shutil.rmtree.assert_called_with(path) + + @unittest.skip('not handled correctly') + @mock.patch('armada.utils.source.shutil') + @mock.patch('armada.utils.source.path') + def test_source_cleanup_bad_path(self, mock_path, mock_shutil): + mock_path.exists.return_value = False + path = 'armada' + with self.assertRaises(Exception): + source.source_cleanup(path) + mock_shutil.rmtree.assert_not_called() diff --git a/armada/utils/git.py b/armada/utils/git.py deleted file mode 100644 index 6f97a92c..00000000 --- a/armada/utils/git.py +++ /dev/null @@ -1,30 +0,0 @@ -import pygit2 -import tempfile -import shutil -from os import path - -from ..exceptions import git_exceptions - -def git_clone(repo_url, branch='master'): - ''' - clones repo to a /tmp/ dir - ''' - - if repo_url == '': - raise git_exceptions.GitLocationException(repo_url) - - _tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp') - - try: - pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch) - except Exception: - raise git_exceptions.GitLocationException(repo_url) - - return _tmp_dir - -def source_cleanup(target_dir): - ''' - Clean up source - ''' - if path.exists(target_dir): - shutil.rmtree(target_dir) diff --git a/armada/utils/source.py b/armada/utils/source.py new file mode 100644 index 00000000..f560e5e0 --- /dev/null +++ b/armada/utils/source.py @@ -0,0 +1,65 @@ +import pygit2 +import requests +import tarfile +import tempfile +import shutil +from os import path + +from ..exceptions import source_exceptions + +def git_clone(repo_url, branch='master'): + ''' + clones repo to a /tmp/ dir + ''' + + if repo_url == '': + raise source_exceptions.GitLocationException(repo_url) + + _tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp') + + try: + pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch) + except Exception: + raise source_exceptions.GitLocationException(repo_url) + + return _tmp_dir + +def get_tarball(tarball_url): + tarball_path = download_tarball(tarball_url) + return extract_tarball(tarball_path) + +def download_tarball(tarball_url): + ''' + Downloads a tarball to /tmp and returns the path + ''' + try: + tarball_filename = tempfile.mkstemp(prefix='armada', dir='/tmp')[1] + response = requests.get(tarball_url) + with open(tarball_filename, 'wb') as f: + f.write(response.content) + except Exception: + raise source_exceptions.TarballDownloadException(tarball_url) + return tarball_filename + +def extract_tarball(tarball_path): + ''' + Extracts a tarball to /tmp and returns the path + ''' + if not path.exists(tarball_path): + raise source_exceptions.InvalidPathException(tarball_path) + + _tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp') + + try: + file = tarfile.open(tarball_path) + file.extractall(_tmp_dir) + except Exception: + raise source_exceptions.TarballExtractException(tarball_path) + return _tmp_dir + +def source_cleanup(target_dir): + ''' + Clean up source + ''' + if path.exists(target_dir): + shutil.rmtree(target_dir) diff --git a/docs/source/operations/guide-build-armada-yaml.rst b/docs/source/operations/guide-build-armada-yaml.rst index 83cf2133..ede90f3f 100644 --- a/docs/source/operations/guide-build-armada-yaml.rst +++ b/docs/source/operations/guide-build-armada-yaml.rst @@ -151,7 +151,7 @@ Source +-------------+----------+---------------------------------------------------------------+ | keyword | type | action | +=============+==========+===============================================================+ -| type | string | source to build the chart: ``git`` or ``local`` | +| type | string | source to build the chart: ``git``, ``local``, or ``tar`` | +-------------+----------+---------------------------------------------------------------+ | location | string | ``url`` or ``path`` to the chart's parent directory | +-------------+----------+---------------------------------------------------------------+ @@ -247,9 +247,9 @@ Simple Example namespace: default values: {} source: - type: git - location: https://github.com/namespace/repo - subpath: . + type: tar + location: http://localhost:8879/namespace/repo + subpath: blog-2 reference: master dependencies: [] --- diff --git a/docs/source/operations/guide-exceptions.rst b/docs/source/operations/guide-exceptions.rst index b0dbd4c1..e4bd7600 100644 --- a/docs/source/operations/guide-exceptions.rst +++ b/docs/source/operations/guide-exceptions.rst @@ -4,8 +4,6 @@ Armada Exceptions +------------------------+----------------------------------------------------------+ | Exception | Error Description | +========================+==========================================================+ -| ChartSourceException | Occurs when an unknown chart source type is encountered. | -+------------------------+----------------------------------------------------------+ | KnownReleasesException | Occurs when no known releases are found. | +------------------------+----------------------------------------------------------+ @@ -53,16 +51,24 @@ Chartbuilder Exceptions | UnknownChartSourceException | The chart source is unknown or invalid. | +-----------------------------+-------------------------------------------------------------+ -Git Exceptions +Source Exceptions =============== -+------------------------+---------------------------------------------+ -| Exception | Error Description | -+========================+=============================================+ -| GitLocationException | Repository location is not valid. | -+------------------------+---------------------------------------------+ -| SourceCleanupException | The source dir of a chart no longer exists. | -+------------------------+---------------------------------------------+ ++--------------------------+---------------------------------------------------------------------+ +| Exception | Error Description | ++==========================+=====================================================================+ +| GitLocationException | Repository location is not valid. | ++--------------------------+---------------------------------------------------------------------+ +| SourceCleanupException | The source dir of a chart no longer exists. | ++--------------------------+---------------------------------------------------------------------+ +| TarballDownloadException | Occurs when the tarball cannot be downloaded from the provided URL. | ++--------------------------+---------------------------------------------------------------------+ +| TarballExtractException | Occurs when extracting a tarball fails. | ++--------------------------+---------------------------------------------------------------------+ +| InvalidPathException | Occurs when a nonexistant path is accessed. | ++--------------------------+---------------------------------------------------------------------+ +| ChartSourceException | Occurs when an unknown chart source type is encountered. | ++--------------------------+---------------------------------------------------------------------+ Lint Exceptions =============== diff --git a/examples/tar_example.yaml b/examples/tar_example.yaml new file mode 100644 index 00000000..c3aeaa7f --- /dev/null +++ b/examples/tar_example.yaml @@ -0,0 +1,57 @@ +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: helm-toolkit +data: + chart_name: helm-toolkit + release: helm-toolkit + namespace: helm-tookit + values: {} + source: + type: git + location: git://github.com/openstack/openstack-helm + subpath: helm-toolkit + reference: master + dependencies: [] +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: mariadb-tarball +data: + chart_name: mariadb-tarball + release: mariadb-tarball + namespace: tarball + timeout: 3600 + install: + no_hooks: false + upgrade: + no_hooks: false + values: {} + source: + type: tar + location: http://localhost:8879/charts/mariadb-0.1.0.tgz + subpath: mariadb + reference: null + dependencies: + - helm-toolkit +--- +schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: tar-example +data: + description: "Deploying mariadb tarball URL" + sequenced: True + chart_group: + - mariadb-tarball +--- +schema: armada/Manifest/v1 +metadata: + schema: metadata/Document/v1 + name: armada-manifest +data: + release_prefix: armada + chart_groups: + - tar-example