From 20dcaa45ae9392e4c8b4ef42bcde476ff784865c Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Fri, 13 Jul 2018 18:35:22 +0100 Subject: [PATCH] Add git and branch revision support to pegleg * Add support for URLs and directories including git clone support * Add support for http://, https://, and ssh:// git cloning * Add support for cloning behind proxy * Add support for checking out references of cloned repos * Add support for checking out references of local repos * Add support for Pegleg Git exceptions This patch set also adds support for including Pegleg source code in documentation and adds exceptions documentation. Change-Id: I417a62c815f97a70f3abc432cc342707e8ce1f54 --- doc/source/conf.py | 8 +- doc/source/exceptions.rst | 67 +++ doc/source/index.rst | 38 +- src/bin/pegleg/pegleg/engine/exceptions.py | 79 ++++ src/bin/pegleg/pegleg/engine/util/__init__.py | 1 + src/bin/pegleg/pegleg/engine/util/git.py | 286 ++++++++++++ src/bin/pegleg/requirements.txt | 1 + .../pegleg/tests/unit/engine/util/__init__.py | 0 .../pegleg/tests/unit/engine/util/test_git.py | 432 ++++++++++++++++++ src/bin/pegleg/tests/unit/fixtures.py | 10 +- src/bin/pegleg/tests/unit/test_utils.py | 39 ++ tox.ini | 5 +- 12 files changed, 940 insertions(+), 26 deletions(-) create mode 100644 doc/source/exceptions.rst create mode 100644 src/bin/pegleg/pegleg/engine/exceptions.py create mode 100644 src/bin/pegleg/pegleg/engine/util/git.py create mode 100644 src/bin/pegleg/tests/unit/engine/util/__init__.py create mode 100644 src/bin/pegleg/tests/unit/engine/util/test_git.py create mode 100644 src/bin/pegleg/tests/unit/test_utils.py diff --git a/doc/source/conf.py b/doc/source/conf.py index dcf30933..caa35783 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -16,9 +16,9 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('../../src/bin/pegleg')) import sphinx_rtd_theme @@ -52,7 +52,7 @@ master_doc = 'index' # General information about the project. project = u'pegleg' copyright = u'2018 AT&T Intellectual Property.' -author = u'pegleg Authors' +author = u'Pegleg Authors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/doc/source/exceptions.rst b/doc/source/exceptions.rst new file mode 100644 index 00000000..afa83f0c --- /dev/null +++ b/doc/source/exceptions.rst @@ -0,0 +1,67 @@ +.. + Copyright 2018 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Pegleg Exceptions +================== + +Base Exceptions +--------------- + +.. list-table:: + :widths: 5 50 + :header-rows: 1 + + * - Exception Name + - Description + * - PeglegBaseException + - .. autoexception:: pegleg.engine.exceptions.PeglegBaseException + :members: + :undoc-members: + +Git Exceptions +-------------- + +.. list-table:: + :widths: 5 50 + :header-rows: 1 + + * - Exception Name + - Description + * - BaseGitException + - .. autoexception:: pegleg.engine.exceptions.BaseGitException + :members: + :show-inheritance: + :undoc-members: + * - GitException + - .. autoexception:: pegleg.engine.exceptions.GitException + :members: + :show-inheritance: + :undoc-members: + * - GitAuthException + - .. autoexception:: pegleg.engine.exceptions.GitAuthException + :members: + :show-inheritance: + :undoc-members: + * - GitProxyException + - .. autoexception:: pegleg.engine.exceptions.GitProxyException + :members: + :show-inheritance: + :undoc-members: + * - GitSSHException + - .. autoexception:: pegleg.engine.exceptions.GitSSHException + :members: + :show-inheritance: + :undoc-members: diff --git a/doc/source/index.rst b/doc/source/index.rst index d4f879a1..07299691 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,26 +14,32 @@ License for the specific language governing permissions and limitations under the License. -.. tip:: +==================== +Pegleg Documentation +==================== - The Undercloud Platform is part of the AIC CP (AT&T Integrated Cloud - Containerized Platform). More details may be found by using the `Treasuremap`_ - -Building this Documentation ---------------------------- - -Use of ``tox -e docs`` will build an HTML version of this documentation that -can be viewed using a browser at docs/build/index.html on the local filesystem. - -Conventions and Standards -------------------------- +Overview +-------- .. toctree:: :maxdepth: 2 getting_started - authoring_strategy - artifacts - cli -.. _Treasuremap: https://github.com/att-comdev/treasuremap +Design +------ + +.. toctree:: + :maxdepth: 2 + + artifacts + authoring_strategy + +Operator's Guide +---------------- + +.. toctree:: + :maxdepth: 2 + + cli + exceptions diff --git a/src/bin/pegleg/pegleg/engine/exceptions.py b/src/bin/pegleg/pegleg/engine/exceptions.py new file mode 100644 index 00000000..2729f205 --- /dev/null +++ b/src/bin/pegleg/pegleg/engine/exceptions.py @@ -0,0 +1,79 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class PeglegBaseException(Exception): + """Base class for Pegleg exception and error handling.""" + + def __init__(self, message=None, **kwargs): + self.message = message or self.message + try: # nosec + self.message = self.message % kwargs + except Exception: + pass + super(PeglegBaseException, self).__init__(self.message) + + +class BaseGitException(PeglegBaseException): + """Base class for Git exceptions and error handling.""" + + message = 'An unknown error occurred while accessing a chart source.' + + +class GitException(BaseGitException): + """Exception when an error occurs cloning a Git repository.""" + + def __init__(self, location, details=None): + self._message = ('Git exception occurred: [%s] may not be a valid git ' + 'repository' % location) + if details: + self._message += '. Details: %s' % details + + super(GitException, self).__init__(self._message) + + +class GitAuthException(BaseGitException): + """Exception that occurs when authentication fails for cloning a repo.""" + + def __init__(self, repo_url, ssh_key_path): + self._repo_url = repo_url + self._ssh_key_path = ssh_key_path + + self._message = ('Failed to authenticate for repo %s with ssh-key at ' + 'path %s.' % (self._repo_url, self._ssh_key_path)) + + super(GitAuthException, self).__init__(self._message) + + +class GitProxyException(BaseGitException): + """Exception when an error occurs cloning a Git repository + through a proxy.""" + + def __init__(self, location): + self._location = location + self._message = ('Could not resolve proxy [%s].' % self._location) + + super(GitProxyException, self).__init__(self._message) + + +class GitSSHException(BaseGitException): + """Exception that occurs when an SSH key could not be found.""" + + def __init__(self, ssh_key_path): + self._ssh_key_path = ssh_key_path + + self._message = ('Failed to find specified SSH key: %s.' % + (self._ssh_key_path)) + + super(GitSSHException, self).__init__(self._message) diff --git a/src/bin/pegleg/pegleg/engine/util/__init__.py b/src/bin/pegleg/pegleg/engine/util/__init__.py index 71db05b7..7168f047 100644 --- a/src/bin/pegleg/pegleg/engine/util/__init__.py +++ b/src/bin/pegleg/pegleg/engine/util/__init__.py @@ -16,3 +16,4 @@ from . import definition from . import files from . import deckhand +from . import git \ No newline at end of file diff --git a/src/bin/pegleg/pegleg/engine/util/git.py b/src/bin/pegleg/pegleg/engine/util/git.py new file mode 100644 index 00000000..64d63781 --- /dev/null +++ b/src/bin/pegleg/pegleg/engine/util/git.py @@ -0,0 +1,286 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import tempfile +from urllib.parse import urlparse + +from git import exc as git_exc +from git import Git +from git import Repo + +from pegleg.engine import exceptions + +LOG = logging.getLogger(__name__) + +__all__ = [ + 'git_handler', +] + + +def git_handler(repo_url, ref, proxy_server=None, auth_key=None): + """Handle directories that are Git repositories. + + If ``repo_url`` is a valid URL for which a local repository doesn't + exist, then clone ``repo_url`` and checkout the given ``ref``. Otherwise, + treat ``repo_url`` as an already-cloned repository and checkout the given + ``ref``. + + Supported ``ref`` formats include: + + * branch name (e.g. 'master') + * refpath (e.g. 'refs/changes/54/457754/73') + * hexsha (e.g. 'ff5496b9c781918fdc49d79f927323eeef2f5320') + + :param repo_url: URL of remote Git repo or path to local Git repo. If no + local copy exists, clone it. Afterward, check out ``ref`` in the repo. + :param ref: branch, commit or reference in the repo to clone. + :param proxy_server: optional, HTTP proxy to use while cloning the repo. + :param auth_key: If supplied results in using SSH to clone the repository + with the specified key. If the value is None, SSH is not used. + :returns: Path to the cloned repo if a repo was cloned, else absolute + path to ``repo_url``. + :raises ValueError: If ``repo_url`` isn't a valid URL or doesn't begin + with a valid protocol (http, https or ssh) for cloning. + :raises NotADirectoryError: If ``repo_url`` isn't a valid directory path. + + """ + + supported_clone_protocols = ('http', 'https', 'ssh') + + try: + parsed_url = urlparse(repo_url) + except Exception as e: + raise ValueError('repo_url=%s is invalid. Details: %s' % (repo_url, e)) + + if not ref: + raise ValueError('ref=%s must be a non-empty, valid Git ref' % ref) + + if not os.path.exists(repo_url): + # we need to clone the repo_url first since it doesn't exist and then + # checkout the appropriate reference - and return the tmpdir + if parsed_url.scheme in supported_clone_protocols: + return _try_git_clone(repo_url, ref, proxy_server, auth_key) + else: + raise ValueError('repo_url=%s must use one of the following ' + 'protocols: %s' % + (repo_url, ', '.join(supported_clone_protocols))) + + # otherwise, we're dealing with a local directory so although + # we do not need to clone, we may need to process the reference + # by checking that out and returning the directory they passed in + else: + LOG.debug('Treating repo_url=%s as an already-cloned repository. ' + 'Attempting to checkout ref=%s', repo_url, ref) + try: + # get absolute path of what is probably a directory + repo_url = os.path.abspath(repo_url) + except Exception: + msg = "The repo_url=%s is not a valid directory" % repo_url + LOG.error(msg) + raise NotADirectoryError(msg) + + repo = Repo(repo_url) + if repo.is_dirty(): + LOG.warning('The locally cloned repo_url=%s is dirty. ' + 'Cleaning up untracked files.', repo_url) + # Reset the index and working tree to match current ref. + repo.head.reset(index=True, working_tree=True) + + try: + # Check whether the ref exists locally. + LOG.info('Attempting to checkout ref=%s from repo_url=%s locally', + ref, repo_url) + _try_git_checkout(repo, repo_url, ref, fetch=False) + except exceptions.GitException: + # Otherwise, attempt to fetch and checkout the missing ref. + LOG.info('ref=%s not found locally for repo_url=%s, fetching from ' + 'remote', ref, repo_url) + # Allow any errors to bubble up. + _try_git_checkout(repo, repo_url, ref, fetch=True) + + return repo_url + + +def _try_git_clone(repo_url, ref='master', proxy_server=None, auth_key=None): + """Try cloning Git repo from ``repo_url`` using the reference ``ref``. + + :param repo_url: URL of remote Git repo or path to local Git repo. + :param ref: branch, commit or reference in the repo to clone. Default is + 'master'. + :param proxy_server: optional, HTTP proxy to use while cloning the repo. + :param auth_key: If supplied results in using SSH to clone the repository + with the specified key. If the value is None, SSH is not used. + :returns: Path to the cloned repo. + :rtype: str + :raises GitException: If ``repo_url`` is invalid or could not be found. + :raises GitAuthException: If authentication with the Git repository failed. + :raises GitProxyException: If the repo could not be cloned due to a proxy + issue. + + """ + + # the name here is important as it bubbles back up to the output filename + # and ensure we handle url/foo.git/ cases. prefix is 'tmp' by default. + temp_dir = tempfile.mkdtemp(suffix=repo_url.rstrip('/').split('/')[-1]) + env_vars = _get_clone_env_vars(repo_url, ref, auth_key) + ssh_cmd = env_vars.get('GIT_SSH_COMMAND') + + try: + if proxy_server: + LOG.debug('Cloning [%s] with proxy [%s]', repo_url, proxy_server) + # TODO(felipemonteiro): proxy_server can be finicky. Need a config + # option to retry up to N times. + repo = Repo.clone_from( + repo_url, + temp_dir, + config='http.proxy=%s' % proxy_server, + env=env_vars) + else: + LOG.debug('Cloning [%s]', repo_url) + repo = Repo.clone_from(repo_url, temp_dir, env=env_vars) + except git_exc.GitCommandError as e: + LOG.exception('Failed to clone repo_url=%s using ref=%s.', repo_url, + ref) + if (ssh_cmd and ssh_cmd in e.stderr + or 'permission denied' in e.stderr.lower()): + raise exceptions.GitAuthException(repo_url, auth_key) + elif 'could not resolve proxy' in e.stderr.lower(): + raise exceptions.GitProxyException(proxy_server) + else: + raise exceptions.GitException(repo_url, details=e) + except Exception as e: + msg = 'Encountered unknown Exception during clone of %s' % repo_url + LOG.exception(msg) + raise exceptions.GitException(repo_url, details=e) + + _try_git_checkout(repo=repo, repo_url=repo_url, ref=ref) + + return temp_dir + + +def _get_clone_env_vars(repo_url, ref, auth_key): + """Generate environment variables include SSH command for Git clone. + + :param repo_url: URL of remote Git repo or path to local Git repo. + :param ref: branch, commit or reference in the repo to clone. Default is + 'master'. + :param auth_key: If supplied results in using SSH to clone the repository + with the specified key. If the value is None, SSH is not used. + :returns: Dictionary of key-value pairs for Git clone. + :rtype: dict + :raises GitSSHException: If the SSH key specified by ``CONF.ssh_key_path`` + could not be found and ``auth_method`` is "SSH". + + """ + ssh_cmd = None + env_vars = {'GIT_TERMINAL_PROMPT': '0'} + + if auth_key: + if os.path.exists(auth_key): + LOG.debug('Attempting to clone the repo at %s using reference %s ' + 'with SSH authentication.', repo_url, ref) + # Ensure that host checking is ignored, to avoid unnecessary + # required CLI input. + ssh_cmd = ( + 'ssh -i {} -o ConnectionAttempts=20 -o ConnectTimeout=10 -o ' + 'StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' + .format(os.path.expanduser(auth_key))) + env_vars.update({'GIT_SSH_COMMAND': ssh_cmd}) + else: + msg = "The auth_key path '%s' was not found" % auth_key + LOG.error(msg) + raise exceptions.GitSSHException(auth_key) + return env_vars + + +def _try_git_checkout(repo, repo_url, ref, fetch=True): + """Try to checkout a ``ref`` from ``repo``. + + Local branches are created for multiple variations of the ``ref``, + including its refpath and hexpath (i.e. commit ID). + + This is to locally "memoize" references that would otherwise require + resolution upstream. We increase performance by creating local branches + for these other ``ref`` formats when the ``ref`` is fetched remotely for + the first time only. + + :param repo: Git Repo object. + :param repo_url: URL of remote Git repo or path to local Git repo. + :param ref: branch, commit or reference in the repo to clone. Default is + 'master'. + :param fetch: Whether to fetch the ``ref`` from remote before checkout or + to use the already-cloned local repo. + :raises GitException: If ``ref`` could not be checked out. + + """ + try: + g = Git(repo.working_dir) + branches = [b.name for b in repo.branches] + LOG.debug('Available branches for repo_url=%s: %s', repo_url, branches) + + if fetch: + LOG.debug('Fetching ref=%s from remote repo_url=%s', ref, repo_url) + # fetch_info is guaranteed to be populated if ref resolves, else + # a GitCommandError is raised. + fetch_info = repo.remotes.origin.fetch(ref) + hexsha = fetch_info[0].commit.hexsha.strip() + ref_path = fetch_info[0].remote_ref_path.strip() + + # If ``ref`` doesn't match the hexsha/refpath then create a branch + # for each so that future checkouts can be performed using either + # format. This way, no future processing is required to figure + # out whether a refpath/hexsha exists within the repo. + _create_local_ref( + g, branches, ref=ref, newref=hexsha, reftype='hexsha') + _create_local_ref( + g, branches, ref=ref, newref=ref_path, reftype='refpath') + _create_or_checkout_local_ref(g, branches, ref=ref) + else: + LOG.debug('Checking out ref=%s from local repo_url=%s', ref, + repo_url) + # Expect the reference to exist if checking out locally. + g.checkout(ref) + + LOG.debug('Successfully checked out ref=%s for repo_url=%s', ref, + repo_url) + except git_exc.GitCommandError as e: + LOG.exception('Failed to checkout ref=%s from repo_url=%s.', ref, + repo_url) + raise exceptions.GitException(repo_url, details=e) + except Exception as e: + msg = ('Encountered unknown Exception during checkout of ref=%s for ' + 'repo_url=%s' % (ref, repo_url)) + LOG.exception(msg) + raise exceptions.GitException(repo_url, details=e) + + +def _create_or_checkout_local_ref(g, branches, ref): + if ref not in branches: + LOG.debug('Creating local branch for ref=%s', ref) + g.checkout('FETCH_HEAD', b=ref) + branches.append(ref) + else: + LOG.debug('Checking out ref=%s from local repo', ref) + g.checkout('FETCH_HEAD') + + +def _create_local_ref(g, branches, ref, newref, reftype=None): + if newref not in branches: + if newref and ref != newref: + LOG.debug('Creating local branch for ref=%s (%s for %s)', newref, + reftype, ref) + g.checkout('FETCH_HEAD', b=newref) + branches.append(newref) diff --git a/src/bin/pegleg/requirements.txt b/src/bin/pegleg/requirements.txt index 8baecc6a..5826d75e 100644 --- a/src/bin/pegleg/requirements.txt +++ b/src/bin/pegleg/requirements.txt @@ -1,3 +1,4 @@ +gitpython click==6.7 jsonschema==2.6.0 pyyaml==3.12 diff --git a/src/bin/pegleg/tests/unit/engine/util/__init__.py b/src/bin/pegleg/tests/unit/engine/util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/bin/pegleg/tests/unit/engine/util/test_git.py b/src/bin/pegleg/tests/unit/engine/util/test_git.py new file mode 100644 index 00000000..19c4f53c --- /dev/null +++ b/src/bin/pegleg/tests/unit/engine/util/test_git.py @@ -0,0 +1,432 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import socket +import requests + +import fixtures +import mock +import pytest + +from pegleg.engine import exceptions +from pegleg.engine.util import git +from tests.unit import test_utils + +_REPO_DIR = None +_PROXY_SERVERS = { + 'http': + os.getenv('HTTP_PROXY', + os.getenv('http_proxy', 'http://one.proxy.att.com:8888')), + 'https': + os.getenv('HTTPS_PROXY', + os.getenv('https_proxy', 'https://one.proxy.att.com:8888')) +} + + +def is_connected(): + """Verifies whether network connectivity is up. + + :returns: True if connected else False. + """ + try: + r = requests.get("http://www.github.com/", proxies={}) + return r.ok + except requests.exceptions.RequestException: + return False + + +def is_connected_behind_proxy(): + """Verifies whether network connectivity is up behind given proxy. + + :returns: True if connected else False. + """ + try: + r = requests.get("http://www.github.com/", proxies=_PROXY_SERVERS) + return r.ok + except requests.exceptions.RequestException: + return False + + +@pytest.fixture() +def clean_git_repo(): + global _REPO_DIR + + try: + yield + finally: + if _REPO_DIR and os.path.exists(_REPO_DIR): + shutil.rmtree(_REPO_DIR) + _REPO_DIR = None + + +def _validate_git_clone(repo_dir, fetched_ref=None, checked_out_ref=None): + """Validate that git clone/checkout work. + + :param repo_dir: Path to local Git repo. + :param fetched_ref: Reference that is stored in FETCH_HEAD following a + remote fetch. + :param checked_out_ref: Reference that is stored in HEAD following a local + ref checkout. + """ + global _REPO_DIR + _REPO_DIR = repo_dir + + assert os.path.isdir(repo_dir) + # Assert that the directory is a Git repo. + assert os.path.isdir(os.path.join(repo_dir, '.git')) + if fetched_ref: + # Assert the FETCH_HEAD is at the fetched_ref ref. + with open(os.path.join(repo_dir, '.git', 'FETCH_HEAD'), 'r') \ + as git_file: + assert fetched_ref in git_file.read() + if checked_out_ref: + # Assert the HEAD is at the checked_out_ref. + with open(os.path.join(repo_dir, '.git', 'HEAD'), 'r') \ + as git_file: + assert checked_out_ref in git_file.read() + + +def _assert_repo_url_was_cloned(mock_log, git_dir): + expected_msg = ('Treating repo_url=%s as an already-cloned repository') + assert mock_log.debug.called + mock_calls = mock_log.debug.mock_calls + assert any(m[1][0].startswith(expected_msg) for m in mock_calls) + assert any(m[1][1] == git_dir for m in mock_calls) + mock_log.debug.reset_mock() + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +def test_git_clone_valid_url_http_protocol(clean_git_repo): + url = 'http://github.com/openstack/airship-armada' + git_dir = git.git_handler(url, ref='master') + _validate_git_clone(git_dir) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +def test_git_clone_valid_url_https_protocol(clean_git_repo): + url = 'https://github.com/openstack/airship-armada' + git_dir = git.git_handler(url, ref='master') + _validate_git_clone(git_dir) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +def test_git_clone_with_commit_reference(clean_git_repo): + url = 'https://github.com/openstack/airship-armada' + commit = 'cba78d1d03e4910f6ab1691bae633c5bddce893d' + git_dir = git.git_handler(url, commit) + _validate_git_clone(git_dir, commit) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +def test_git_clone_with_patch_ref(clean_git_repo): + ref = 'refs/changes/54/457754/73' + git_dir = git.git_handler('https://github.com/openstack/openstack-helm', + ref) + _validate_git_clone(git_dir, ref) + + +@pytest.mark.skipif( + not is_connected_behind_proxy(), + reason='git clone requires proxy connectivity.') +@mock.patch.object(git, 'LOG', autospec=True) +def test_git_clone_behind_proxy(mock_log, clean_git_repo): + url = 'https://github.com/openstack/airship-armada' + commit = 'cba78d1d03e4910f6ab1691bae633c5bddce893d' + + for proxy_server in _PROXY_SERVERS.values(): + git_dir = git.git_handler(url, commit, proxy_server=proxy_server) + _validate_git_clone(git_dir, commit) + + mock_log.debug.assert_any_call('Cloning [%s] with proxy [%s]', url, + proxy_server) + mock_log.debug.reset_mock() + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +@mock.patch.object(git, 'LOG', autospec=True) +def test_git_clone_existing_directory_checks_out_earlier_ref_from_local( + mock_log, clean_git_repo): + """Validate Git checks out an earlier patch or ref that should exist + locally (as a later ref was already fetched which should contain that + revision history). + """ + # Clone the openstack-helm repo and automatically checkout patch 34. + ref = 'refs/changes/15/536215/35' + repo_url = 'https://github.com/openstack/openstack-helm' + git_dir = git.git_handler(repo_url, ref) + _validate_git_clone(git_dir, fetched_ref=ref) + + # Checkout ref='master' now that the repo already exists locally. + ref = 'refs/changes/15/536215/34' + git_dir = git.git_handler(git_dir, ref) + _validate_git_clone(git_dir, checked_out_ref=ref) + _assert_repo_url_was_cloned(mock_log, git_dir) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +@mock.patch.object(git, 'LOG', autospec=True) +def test_git_clone_existing_directory_checks_out_master_from_local( + mock_log, clean_git_repo): + """Validate Git checks out the ref of an already cloned repo that exists + locally. + """ + # Clone the openstack-helm repo and automatically checkout patch 34. + ref = 'refs/changes/15/536215/34' + repo_url = 'https://github.com/openstack/openstack-helm' + git_dir = git.git_handler(repo_url, ref) + _validate_git_clone(git_dir, fetched_ref=ref) + + # Checkout ref='master' now that the repo already exists locally. + ref = 'master' + git_dir = git.git_handler(git_dir, ref) + _validate_git_clone(git_dir, checked_out_ref=ref) + _assert_repo_url_was_cloned(mock_log, git_dir) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +@mock.patch.object(git, 'LOG', autospec=True) +def test_git_clone_checkout_refpath_saves_references_locally( + mock_log, clean_git_repo): + """Validate that refpath/hexsha branches are created in the local repo + following clone of the repo using a refpath during initial checkout. + """ + # Clone the openstack-helm repo and automatically checkout patch 34. + ref = 'refs/changes/15/536215/34' + repo_url = 'https://github.com/openstack/openstack-helm' + git_dir = git.git_handler(repo_url, ref) + _validate_git_clone(git_dir, fetched_ref=ref) + + # Now checkout patch 34 again to ensure it's still there. + ref = 'refs/changes/15/536215/34' + git_dir = git.git_handler(git_dir, ref) + _validate_git_clone(git_dir, checked_out_ref=ref) + _assert_repo_url_was_cloned(mock_log, git_dir) + + # Verify that passing in the hexsha variation of refpath + # 'refs/changes/15/536215/34' also works. + hexref = '276102a115dac3c0a6e91f9047d8b086bc8d2ff0' + git_dir = git.git_handler(git_dir, hexref) + _validate_git_clone(git_dir, checked_out_ref=hexref) + _assert_repo_url_was_cloned(mock_log, git_dir) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +@mock.patch.object(git, 'LOG', autospec=True) +def test_git_clone_checkout_hexsha_saves_references_locally( + mock_log, clean_git_repo): + """Validate that refpath/hexsha branches are created in the local repo + following clone of the repo using a hexsha during initial checkout. + """ + # Clone the openstack-helm repo and automatically checkout patch using + # hexsha. + # NOTE(felipemonteiro): We have to use the commit ID (hexsha) corresponding + # to the last patch as that is what gets pushed to github. In this case, + # this corresponds to patch 'refs/changes/15/536215/35'. + ref = 'bf126f46b1c175a8038949a87dafb0a716e3b9b6' + repo_url = 'https://github.com/openstack/openstack-helm' + git_dir = git.git_handler(repo_url, ref) + _validate_git_clone(git_dir, fetched_ref=ref) + + # Now checkout patch using hexsha again to ensure it's still there. + ref = 'bf126f46b1c175a8038949a87dafb0a716e3b9b6' + git_dir = git.git_handler(git_dir, ref) + _validate_git_clone(git_dir, checked_out_ref=ref) + _assert_repo_url_was_cloned(mock_log, git_dir) + + # Verify that passing in the refpath variation of hexsha also works. + hexref = 'refs/changes/15/536215/35' + git_dir = git.git_handler(git_dir, hexref) + _validate_git_clone(git_dir, checked_out_ref=hexref) + _assert_repo_url_was_cloned(mock_log, git_dir) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +@mock.patch.object(git, 'LOG', autospec=True) +def test_git_clone_existing_directory_checks_out_next_local_ref( + mock_log, clean_git_repo): + """Validate Git fetches the newer ref upstream that doesn't exist locally + in the cloned repo. + """ + # Clone the openstack-helm repo and automatically checkout patch 73. + ref = 'refs/changes/54/457754/73' + repo_url = 'https://github.com/openstack/openstack-helm' + git_dir = git.git_handler(repo_url, ref) + _validate_git_clone(git_dir, ref) + + # Attempt to checkout patch 74 which requires a remote fetch even though + # the repo has already been cloned. + ref = 'refs/changes/54/457754/74' + git_dir = git.git_handler(git_dir, ref) + _validate_git_clone(git_dir, ref) + _assert_repo_url_was_cloned(mock_log, git_dir) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +@mock.patch.object(git, 'LOG', autospec=True) +def test_git_clone_delete_repo_and_reclone(mock_log, clean_git_repo): + """Validate that cloning a repo, then deleting it, then recloning it works. + """ + # Clone the openstack-helm repo and automatically checkout patch 73. + ref = 'refs/changes/54/457754/73' + repo_url = 'https://github.com/openstack/openstack-helm' + first_git_dir = git.git_handler(repo_url, ref) + _validate_git_clone(first_git_dir, ref) + + # Validate that the repo was cloned. + assert mock_log.debug.called + mock_log.debug.assert_any_call('Cloning [%s]', repo_url) + mock_log.debug.reset_mock() + + # Delete the just-cloned repo. + shutil.rmtree(first_git_dir) + + # Verify that checking out the same ref results in a re-clone. + second_git_dir = git.git_handler(repo_url, ref) + _validate_git_clone(second_git_dir, ref) + + # Validate that the repo was cloned. + assert first_git_dir != second_git_dir + assert mock_log.debug.called + mock_log.debug.assert_any_call('Cloning [%s]', repo_url) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +@mock.patch.object(git, 'LOG', autospec=True) +def test_git_clone_clean_dirty_local_repo(mock_log, clean_git_repo): + """Validate that a dirty repo is cleaned before a ref is checked out.""" + ref = 'refs/changes/54/457754/73' + repo_url = 'https://github.com/openstack/openstack-helm' + git_dir = git.git_handler(repo_url, ref) + _validate_git_clone(git_dir, ref) + + file_to_rename = os.path.join(git_dir, os.listdir(git_dir)[0]) + os.rename(file_to_rename, file_to_rename + '-renamed') + + git_dir = git.git_handler(git_dir, ref) + _validate_git_clone(git_dir, ref) + + assert mock_log.warning.called + mock_log.warning.assert_any_call( + 'The locally cloned repo_url=%s is dirty. Cleaning up untracked ' + 'files.', git_dir) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +@mock.patch.object(git, 'LOG', autospec=True) +def test_git_clone_existing_directory_raises_exc_for_invalid_ref( + mock_log, clean_git_repo): + """Validate Git throws an error for an invalid ref when trying to checkout + a ref for an already-cloned repo. + """ + # Clone the openstack-helm repo and automatically checkout patch 73. + ref = 'refs/changes/54/457754/73' + repo_url = 'https://github.com/openstack/openstack-helm' + git_dir = git.git_handler(repo_url, ref) + _validate_git_clone(git_dir, ref) + + # Attempt to checkout patch 9000 now that the repo already exists locally. + ref = 'refs/changes/54/457754/9000' + with pytest.raises(exceptions.GitException): + git_dir = git.git_handler(git_dir, ref) + _assert_repo_url_was_cloned(mock_log, git_dir) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +def test_git_clone_empty_url_raises_value_error(clean_git_repo): + url = '' + with pytest.raises(ValueError): + git.git_handler(url, ref='master') + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +def test_git_clone_invalid_url_type_raises_value_error(clean_git_repo): + url = 5 + with pytest.raises(ValueError): + git.git_handler(url, ref='master') + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +def test_git_clone_invalid_local_repo_url_raises_notadirectory_error( + clean_git_repo): + url = False + with pytest.raises(NotADirectoryError): + git.git_handler(url, ref='master') + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +def test_git_clone_invalid_remote_url(clean_git_repo): + url = 'https://github.com/dummy/armada' + with pytest.raises(exceptions.GitException): + git.git_handler(url, ref='master') + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +def test_git_clone_invalid_remote_url_protocol(clean_git_repo): + url = 'ftp://foo.bar' + with pytest.raises(ValueError): + git.git_handler(url, ref='master') + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +def test_git_clone_fake_proxy(clean_git_repo): + url = 'https://github.com/openstack/airship-armada' + proxy_url = test_utils.rand_name( + 'not.a.proxy.that.works.and.never.will', prefix='http://') + ":8080" + + with pytest.raises(exceptions.GitProxyException): + git.git_handler(url, ref='master', proxy_server=proxy_url) + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +@mock.patch('os.path.exists', return_value=True, autospec=True) +def test_git_clone_ssh_auth_method_fails_auth(_, clean_git_repo): + fake_user = test_utils.rand_name('fake_user') + url = ('ssh://%s@review.openstack.org:29418/openstack/airship-armada' % + fake_user) + with pytest.raises(exceptions.GitAuthException): + git._try_git_clone( + url, ref='refs/changes/17/388517/5', auth_key='/home/user/.ssh/') + + +@pytest.mark.skipif( + not is_connected(), reason='git clone requires network connectivity.') +@mock.patch('os.path.exists', return_value=False, autospec=True) +def test_git_clone_ssh_auth_method_missing_ssh_key(_, clean_git_repo): + fake_user = test_utils.rand_name('fake_user') + url = ('ssh://%s@review.openstack.org:29418/openstack/airship-armada' % + fake_user) + with pytest.raises(exceptions.GitSSHException): + git.git_handler( + url, ref='refs/changes/17/388517/5', auth_key='/home/user/.ssh/') diff --git a/src/bin/pegleg/tests/unit/fixtures.py b/src/bin/pegleg/tests/unit/fixtures.py index d7b3866e..8f33dfa7 100644 --- a/src/bin/pegleg/tests/unit/fixtures.py +++ b/src/bin/pegleg/tests/unit/fixtures.py @@ -47,6 +47,9 @@ def _gen_document(**kwargs): def create_tmp_deployment_files(tmpdir): """Fixture that creates a temporary directory structure.""" sitenames = ['cicd', 'lab'] + # Used for ensuring the original global context is reset in memory + # following each test execution. + original_global_context = copy.deepcopy(config.GLOBAL_CONTEXT) SITE_TEST_STRUCTURE = { 'directories': { @@ -152,8 +155,5 @@ schema: pegleg/SiteDefinition/v1 yield - config.GLOBAL_CONTEXT = { - 'primary_repo': './', - 'aux_repos': [], - 'site_path': 'site' - } + # Restore the global context back to blank slate status. + config.GLOBAL_CONTEXT = original_global_context diff --git a/src/bin/pegleg/tests/unit/test_utils.py b/src/bin/pegleg/tests/unit/test_utils.py new file mode 100644 index 00000000..9412c642 --- /dev/null +++ b/src/bin/pegleg/tests/unit/test_utils.py @@ -0,0 +1,39 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2015 Hewlett-Packard Development Company, L.P. +# Copyright 2017 AT&T Intellectual Property. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import random +import uuid + + +def rand_name(name='', prefix='armada'): + """Generate a random name that includes a random number + + :param str name: The name that you want to include + :param str prefix: The prefix that you want to include + :return: a random name. The format is + '--'. + (e.g. 'prefixfoo-namebar-154876201') + :rtype: string + """ + randbits = str(random.randint(1, 0x7fffffff)) + rand_name = randbits + if name: + rand_name = name + '-' + rand_name + if prefix: + rand_name = prefix + '-' + rand_name + return rand_name diff --git a/tox.ini b/tox.ini index 94375228..b98feaea 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,10 @@ commands = whitelist_externals = tox [testenv:docs] -deps = -r{toxinidir}/doc/requirements.txt +basepython = python3 +deps = + -r{toxinidir}/src/bin/pegleg/requirements.txt + -r{toxinidir}/doc/requirements.txt commands = rm -rf doc/build sphinx-build -b html doc/source doc/build -n -W -v