Add git and branch revision support to pegleg

* Add support for URLs and directories including git clone support
* Add support for http://, https://, and ssh:// git cloning
* Add support for cloning behind proxy
* Add support for checking out references of cloned repos
* Add support for checking out references of local repos
* Add support for Pegleg Git exceptions

This patch set also adds support for including Pegleg
source code in documentation and adds exceptions
documentation.

Change-Id: I417a62c815f97a70f3abc432cc342707e8ce1f54
This commit is contained in:
Felipe Monteiro 2018-07-13 18:35:22 +01:00
parent f0ae58d8b9
commit 20dcaa45ae
12 changed files with 940 additions and 26 deletions

View File

@ -16,9 +16,9 @@
# add these directories to sys.path here. If the directory is relative to the # add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here. # documentation root, use os.path.abspath to make it absolute, like shown here.
# #
# import os import os
# import sys import sys
# sys.path.insert(0, os.path.abspath('.')) sys.path.insert(0, os.path.abspath('../../src/bin/pegleg'))
import sphinx_rtd_theme import sphinx_rtd_theme
@ -52,7 +52,7 @@ master_doc = 'index'
# General information about the project. # General information about the project.
project = u'pegleg' project = u'pegleg'
copyright = u'2018 AT&T Intellectual Property.' copyright = u'2018 AT&T Intellectual Property.'
author = u'pegleg Authors' author = u'Pegleg Authors'
# The version info for the project you're documenting, acts as replacement for # The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the # |version| and |release|, also used in various other places throughout the

67
doc/source/exceptions.rst Normal file
View File

@ -0,0 +1,67 @@
..
Copyright 2018 AT&T Intellectual Property.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Pegleg Exceptions
==================
Base Exceptions
---------------
.. list-table::
:widths: 5 50
:header-rows: 1
* - Exception Name
- Description
* - PeglegBaseException
- .. autoexception:: pegleg.engine.exceptions.PeglegBaseException
:members:
:undoc-members:
Git Exceptions
--------------
.. list-table::
:widths: 5 50
:header-rows: 1
* - Exception Name
- Description
* - BaseGitException
- .. autoexception:: pegleg.engine.exceptions.BaseGitException
:members:
:show-inheritance:
:undoc-members:
* - GitException
- .. autoexception:: pegleg.engine.exceptions.GitException
:members:
:show-inheritance:
:undoc-members:
* - GitAuthException
- .. autoexception:: pegleg.engine.exceptions.GitAuthException
:members:
:show-inheritance:
:undoc-members:
* - GitProxyException
- .. autoexception:: pegleg.engine.exceptions.GitProxyException
:members:
:show-inheritance:
:undoc-members:
* - GitSSHException
- .. autoexception:: pegleg.engine.exceptions.GitSSHException
:members:
:show-inheritance:
:undoc-members:

View File

@ -14,26 +14,32 @@
License for the specific language governing permissions and limitations License for the specific language governing permissions and limitations
under the License. under the License.
.. tip:: ====================
Pegleg Documentation
====================
The Undercloud Platform is part of the AIC CP (AT&T Integrated Cloud Overview
Containerized Platform). More details may be found by using the `Treasuremap`_ --------
Building this Documentation
---------------------------
Use of ``tox -e docs`` will build an HTML version of this documentation that
can be viewed using a browser at docs/build/index.html on the local filesystem.
Conventions and Standards
-------------------------
.. toctree:: .. toctree::
:maxdepth: 2 :maxdepth: 2
getting_started getting_started
authoring_strategy
artifacts
cli
.. _Treasuremap: https://github.com/att-comdev/treasuremap Design
------
.. toctree::
:maxdepth: 2
artifacts
authoring_strategy
Operator's Guide
----------------
.. toctree::
:maxdepth: 2
cli
exceptions

View File

@ -0,0 +1,79 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class PeglegBaseException(Exception):
"""Base class for Pegleg exception and error handling."""
def __init__(self, message=None, **kwargs):
self.message = message or self.message
try: # nosec
self.message = self.message % kwargs
except Exception:
pass
super(PeglegBaseException, self).__init__(self.message)
class BaseGitException(PeglegBaseException):
"""Base class for Git exceptions and error handling."""
message = 'An unknown error occurred while accessing a chart source.'
class GitException(BaseGitException):
"""Exception when an error occurs cloning a Git repository."""
def __init__(self, location, details=None):
self._message = ('Git exception occurred: [%s] may not be a valid git '
'repository' % location)
if details:
self._message += '. Details: %s' % details
super(GitException, self).__init__(self._message)
class GitAuthException(BaseGitException):
"""Exception that occurs when authentication fails for cloning a repo."""
def __init__(self, repo_url, ssh_key_path):
self._repo_url = repo_url
self._ssh_key_path = ssh_key_path
self._message = ('Failed to authenticate for repo %s with ssh-key at '
'path %s.' % (self._repo_url, self._ssh_key_path))
super(GitAuthException, self).__init__(self._message)
class GitProxyException(BaseGitException):
"""Exception when an error occurs cloning a Git repository
through a proxy."""
def __init__(self, location):
self._location = location
self._message = ('Could not resolve proxy [%s].' % self._location)
super(GitProxyException, self).__init__(self._message)
class GitSSHException(BaseGitException):
"""Exception that occurs when an SSH key could not be found."""
def __init__(self, ssh_key_path):
self._ssh_key_path = ssh_key_path
self._message = ('Failed to find specified SSH key: %s.' %
(self._ssh_key_path))
super(GitSSHException, self).__init__(self._message)

View File

@ -16,3 +16,4 @@
from . import definition from . import definition
from . import files from . import files
from . import deckhand from . import deckhand
from . import git

View File

@ -0,0 +1,286 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
import tempfile
from urllib.parse import urlparse
from git import exc as git_exc
from git import Git
from git import Repo
from pegleg.engine import exceptions
LOG = logging.getLogger(__name__)
__all__ = [
'git_handler',
]
def git_handler(repo_url, ref, proxy_server=None, auth_key=None):
"""Handle directories that are Git repositories.
If ``repo_url`` is a valid URL for which a local repository doesn't
exist, then clone ``repo_url`` and checkout the given ``ref``. Otherwise,
treat ``repo_url`` as an already-cloned repository and checkout the given
``ref``.
Supported ``ref`` formats include:
* branch name (e.g. 'master')
* refpath (e.g. 'refs/changes/54/457754/73')
* hexsha (e.g. 'ff5496b9c781918fdc49d79f927323eeef2f5320')
:param repo_url: URL of remote Git repo or path to local Git repo. If no
local copy exists, clone it. Afterward, check out ``ref`` in the repo.
:param ref: branch, commit or reference in the repo to clone.
:param proxy_server: optional, HTTP proxy to use while cloning the repo.
:param auth_key: If supplied results in using SSH to clone the repository
with the specified key. If the value is None, SSH is not used.
:returns: Path to the cloned repo if a repo was cloned, else absolute
path to ``repo_url``.
:raises ValueError: If ``repo_url`` isn't a valid URL or doesn't begin
with a valid protocol (http, https or ssh) for cloning.
:raises NotADirectoryError: If ``repo_url`` isn't a valid directory path.
"""
supported_clone_protocols = ('http', 'https', 'ssh')
try:
parsed_url = urlparse(repo_url)
except Exception as e:
raise ValueError('repo_url=%s is invalid. Details: %s' % (repo_url, e))
if not ref:
raise ValueError('ref=%s must be a non-empty, valid Git ref' % ref)
if not os.path.exists(repo_url):
# we need to clone the repo_url first since it doesn't exist and then
# checkout the appropriate reference - and return the tmpdir
if parsed_url.scheme in supported_clone_protocols:
return _try_git_clone(repo_url, ref, proxy_server, auth_key)
else:
raise ValueError('repo_url=%s must use one of the following '
'protocols: %s' %
(repo_url, ', '.join(supported_clone_protocols)))
# otherwise, we're dealing with a local directory so although
# we do not need to clone, we may need to process the reference
# by checking that out and returning the directory they passed in
else:
LOG.debug('Treating repo_url=%s as an already-cloned repository. '
'Attempting to checkout ref=%s', repo_url, ref)
try:
# get absolute path of what is probably a directory
repo_url = os.path.abspath(repo_url)
except Exception:
msg = "The repo_url=%s is not a valid directory" % repo_url
LOG.error(msg)
raise NotADirectoryError(msg)
repo = Repo(repo_url)
if repo.is_dirty():
LOG.warning('The locally cloned repo_url=%s is dirty. '
'Cleaning up untracked files.', repo_url)
# Reset the index and working tree to match current ref.
repo.head.reset(index=True, working_tree=True)
try:
# Check whether the ref exists locally.
LOG.info('Attempting to checkout ref=%s from repo_url=%s locally',
ref, repo_url)
_try_git_checkout(repo, repo_url, ref, fetch=False)
except exceptions.GitException:
# Otherwise, attempt to fetch and checkout the missing ref.
LOG.info('ref=%s not found locally for repo_url=%s, fetching from '
'remote', ref, repo_url)
# Allow any errors to bubble up.
_try_git_checkout(repo, repo_url, ref, fetch=True)
return repo_url
def _try_git_clone(repo_url, ref='master', proxy_server=None, auth_key=None):
"""Try cloning Git repo from ``repo_url`` using the reference ``ref``.
:param repo_url: URL of remote Git repo or path to local Git repo.
:param ref: branch, commit or reference in the repo to clone. Default is
'master'.
:param proxy_server: optional, HTTP proxy to use while cloning the repo.
:param auth_key: If supplied results in using SSH to clone the repository
with the specified key. If the value is None, SSH is not used.
:returns: Path to the cloned repo.
:rtype: str
:raises GitException: If ``repo_url`` is invalid or could not be found.
:raises GitAuthException: If authentication with the Git repository failed.
:raises GitProxyException: If the repo could not be cloned due to a proxy
issue.
"""
# the name here is important as it bubbles back up to the output filename
# and ensure we handle url/foo.git/ cases. prefix is 'tmp' by default.
temp_dir = tempfile.mkdtemp(suffix=repo_url.rstrip('/').split('/')[-1])
env_vars = _get_clone_env_vars(repo_url, ref, auth_key)
ssh_cmd = env_vars.get('GIT_SSH_COMMAND')
try:
if proxy_server:
LOG.debug('Cloning [%s] with proxy [%s]', repo_url, proxy_server)
# TODO(felipemonteiro): proxy_server can be finicky. Need a config
# option to retry up to N times.
repo = Repo.clone_from(
repo_url,
temp_dir,
config='http.proxy=%s' % proxy_server,
env=env_vars)
else:
LOG.debug('Cloning [%s]', repo_url)
repo = Repo.clone_from(repo_url, temp_dir, env=env_vars)
except git_exc.GitCommandError as e:
LOG.exception('Failed to clone repo_url=%s using ref=%s.', repo_url,
ref)
if (ssh_cmd and ssh_cmd in e.stderr
or 'permission denied' in e.stderr.lower()):
raise exceptions.GitAuthException(repo_url, auth_key)
elif 'could not resolve proxy' in e.stderr.lower():
raise exceptions.GitProxyException(proxy_server)
else:
raise exceptions.GitException(repo_url, details=e)
except Exception as e:
msg = 'Encountered unknown Exception during clone of %s' % repo_url
LOG.exception(msg)
raise exceptions.GitException(repo_url, details=e)
_try_git_checkout(repo=repo, repo_url=repo_url, ref=ref)
return temp_dir
def _get_clone_env_vars(repo_url, ref, auth_key):
"""Generate environment variables include SSH command for Git clone.
:param repo_url: URL of remote Git repo or path to local Git repo.
:param ref: branch, commit or reference in the repo to clone. Default is
'master'.
:param auth_key: If supplied results in using SSH to clone the repository
with the specified key. If the value is None, SSH is not used.
:returns: Dictionary of key-value pairs for Git clone.
:rtype: dict
:raises GitSSHException: If the SSH key specified by ``CONF.ssh_key_path``
could not be found and ``auth_method`` is "SSH".
"""
ssh_cmd = None
env_vars = {'GIT_TERMINAL_PROMPT': '0'}
if auth_key:
if os.path.exists(auth_key):
LOG.debug('Attempting to clone the repo at %s using reference %s '
'with SSH authentication.', repo_url, ref)
# Ensure that host checking is ignored, to avoid unnecessary
# required CLI input.
ssh_cmd = (
'ssh -i {} -o ConnectionAttempts=20 -o ConnectTimeout=10 -o '
'StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
.format(os.path.expanduser(auth_key)))
env_vars.update({'GIT_SSH_COMMAND': ssh_cmd})
else:
msg = "The auth_key path '%s' was not found" % auth_key
LOG.error(msg)
raise exceptions.GitSSHException(auth_key)
return env_vars
def _try_git_checkout(repo, repo_url, ref, fetch=True):
"""Try to checkout a ``ref`` from ``repo``.
Local branches are created for multiple variations of the ``ref``,
including its refpath and hexpath (i.e. commit ID).
This is to locally "memoize" references that would otherwise require
resolution upstream. We increase performance by creating local branches
for these other ``ref`` formats when the ``ref`` is fetched remotely for
the first time only.
:param repo: Git Repo object.
:param repo_url: URL of remote Git repo or path to local Git repo.
:param ref: branch, commit or reference in the repo to clone. Default is
'master'.
:param fetch: Whether to fetch the ``ref`` from remote before checkout or
to use the already-cloned local repo.
:raises GitException: If ``ref`` could not be checked out.
"""
try:
g = Git(repo.working_dir)
branches = [b.name for b in repo.branches]
LOG.debug('Available branches for repo_url=%s: %s', repo_url, branches)
if fetch:
LOG.debug('Fetching ref=%s from remote repo_url=%s', ref, repo_url)
# fetch_info is guaranteed to be populated if ref resolves, else
# a GitCommandError is raised.
fetch_info = repo.remotes.origin.fetch(ref)
hexsha = fetch_info[0].commit.hexsha.strip()
ref_path = fetch_info[0].remote_ref_path.strip()
# If ``ref`` doesn't match the hexsha/refpath then create a branch
# for each so that future checkouts can be performed using either
# format. This way, no future processing is required to figure
# out whether a refpath/hexsha exists within the repo.
_create_local_ref(
g, branches, ref=ref, newref=hexsha, reftype='hexsha')
_create_local_ref(
g, branches, ref=ref, newref=ref_path, reftype='refpath')
_create_or_checkout_local_ref(g, branches, ref=ref)
else:
LOG.debug('Checking out ref=%s from local repo_url=%s', ref,
repo_url)
# Expect the reference to exist if checking out locally.
g.checkout(ref)
LOG.debug('Successfully checked out ref=%s for repo_url=%s', ref,
repo_url)
except git_exc.GitCommandError as e:
LOG.exception('Failed to checkout ref=%s from repo_url=%s.', ref,
repo_url)
raise exceptions.GitException(repo_url, details=e)
except Exception as e:
msg = ('Encountered unknown Exception during checkout of ref=%s for '
'repo_url=%s' % (ref, repo_url))
LOG.exception(msg)
raise exceptions.GitException(repo_url, details=e)
def _create_or_checkout_local_ref(g, branches, ref):
if ref not in branches:
LOG.debug('Creating local branch for ref=%s', ref)
g.checkout('FETCH_HEAD', b=ref)
branches.append(ref)
else:
LOG.debug('Checking out ref=%s from local repo', ref)
g.checkout('FETCH_HEAD')
def _create_local_ref(g, branches, ref, newref, reftype=None):
if newref not in branches:
if newref and ref != newref:
LOG.debug('Creating local branch for ref=%s (%s for %s)', newref,
reftype, ref)
g.checkout('FETCH_HEAD', b=newref)
branches.append(newref)

View File

@ -1,3 +1,4 @@
gitpython
click==6.7 click==6.7
jsonschema==2.6.0 jsonschema==2.6.0
pyyaml==3.12 pyyaml==3.12

View File

@ -0,0 +1,432 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import shutil
import socket
import requests
import fixtures
import mock
import pytest
from pegleg.engine import exceptions
from pegleg.engine.util import git
from tests.unit import test_utils
_REPO_DIR = None
_PROXY_SERVERS = {
'http':
os.getenv('HTTP_PROXY',
os.getenv('http_proxy', 'http://one.proxy.att.com:8888')),
'https':
os.getenv('HTTPS_PROXY',
os.getenv('https_proxy', 'https://one.proxy.att.com:8888'))
}
def is_connected():
"""Verifies whether network connectivity is up.
:returns: True if connected else False.
"""
try:
r = requests.get("http://www.github.com/", proxies={})
return r.ok
except requests.exceptions.RequestException:
return False
def is_connected_behind_proxy():
"""Verifies whether network connectivity is up behind given proxy.
:returns: True if connected else False.
"""
try:
r = requests.get("http://www.github.com/", proxies=_PROXY_SERVERS)
return r.ok
except requests.exceptions.RequestException:
return False
@pytest.fixture()
def clean_git_repo():
global _REPO_DIR
try:
yield
finally:
if _REPO_DIR and os.path.exists(_REPO_DIR):
shutil.rmtree(_REPO_DIR)
_REPO_DIR = None
def _validate_git_clone(repo_dir, fetched_ref=None, checked_out_ref=None):
"""Validate that git clone/checkout work.
:param repo_dir: Path to local Git repo.
:param fetched_ref: Reference that is stored in FETCH_HEAD following a
remote fetch.
:param checked_out_ref: Reference that is stored in HEAD following a local
ref checkout.
"""
global _REPO_DIR
_REPO_DIR = repo_dir
assert os.path.isdir(repo_dir)
# Assert that the directory is a Git repo.
assert os.path.isdir(os.path.join(repo_dir, '.git'))
if fetched_ref:
# Assert the FETCH_HEAD is at the fetched_ref ref.
with open(os.path.join(repo_dir, '.git', 'FETCH_HEAD'), 'r') \
as git_file:
assert fetched_ref in git_file.read()
if checked_out_ref:
# Assert the HEAD is at the checked_out_ref.
with open(os.path.join(repo_dir, '.git', 'HEAD'), 'r') \
as git_file:
assert checked_out_ref in git_file.read()
def _assert_repo_url_was_cloned(mock_log, git_dir):
expected_msg = ('Treating repo_url=%s as an already-cloned repository')
assert mock_log.debug.called
mock_calls = mock_log.debug.mock_calls
assert any(m[1][0].startswith(expected_msg) for m in mock_calls)
assert any(m[1][1] == git_dir for m in mock_calls)
mock_log.debug.reset_mock()
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
def test_git_clone_valid_url_http_protocol(clean_git_repo):
url = 'http://github.com/openstack/airship-armada'
git_dir = git.git_handler(url, ref='master')
_validate_git_clone(git_dir)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
def test_git_clone_valid_url_https_protocol(clean_git_repo):
url = 'https://github.com/openstack/airship-armada'
git_dir = git.git_handler(url, ref='master')
_validate_git_clone(git_dir)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
def test_git_clone_with_commit_reference(clean_git_repo):
url = 'https://github.com/openstack/airship-armada'
commit = 'cba78d1d03e4910f6ab1691bae633c5bddce893d'
git_dir = git.git_handler(url, commit)
_validate_git_clone(git_dir, commit)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
def test_git_clone_with_patch_ref(clean_git_repo):
ref = 'refs/changes/54/457754/73'
git_dir = git.git_handler('https://github.com/openstack/openstack-helm',
ref)
_validate_git_clone(git_dir, ref)
@pytest.mark.skipif(
not is_connected_behind_proxy(),
reason='git clone requires proxy connectivity.')
@mock.patch.object(git, 'LOG', autospec=True)
def test_git_clone_behind_proxy(mock_log, clean_git_repo):
url = 'https://github.com/openstack/airship-armada'
commit = 'cba78d1d03e4910f6ab1691bae633c5bddce893d'
for proxy_server in _PROXY_SERVERS.values():
git_dir = git.git_handler(url, commit, proxy_server=proxy_server)
_validate_git_clone(git_dir, commit)
mock_log.debug.assert_any_call('Cloning [%s] with proxy [%s]', url,
proxy_server)
mock_log.debug.reset_mock()
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
@mock.patch.object(git, 'LOG', autospec=True)
def test_git_clone_existing_directory_checks_out_earlier_ref_from_local(
mock_log, clean_git_repo):
"""Validate Git checks out an earlier patch or ref that should exist
locally (as a later ref was already fetched which should contain that
revision history).
"""
# Clone the openstack-helm repo and automatically checkout patch 34.
ref = 'refs/changes/15/536215/35'
repo_url = 'https://github.com/openstack/openstack-helm'
git_dir = git.git_handler(repo_url, ref)
_validate_git_clone(git_dir, fetched_ref=ref)
# Checkout ref='master' now that the repo already exists locally.
ref = 'refs/changes/15/536215/34'
git_dir = git.git_handler(git_dir, ref)
_validate_git_clone(git_dir, checked_out_ref=ref)
_assert_repo_url_was_cloned(mock_log, git_dir)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
@mock.patch.object(git, 'LOG', autospec=True)
def test_git_clone_existing_directory_checks_out_master_from_local(
mock_log, clean_git_repo):
"""Validate Git checks out the ref of an already cloned repo that exists
locally.
"""
# Clone the openstack-helm repo and automatically checkout patch 34.
ref = 'refs/changes/15/536215/34'
repo_url = 'https://github.com/openstack/openstack-helm'
git_dir = git.git_handler(repo_url, ref)
_validate_git_clone(git_dir, fetched_ref=ref)
# Checkout ref='master' now that the repo already exists locally.
ref = 'master'
git_dir = git.git_handler(git_dir, ref)
_validate_git_clone(git_dir, checked_out_ref=ref)
_assert_repo_url_was_cloned(mock_log, git_dir)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
@mock.patch.object(git, 'LOG', autospec=True)
def test_git_clone_checkout_refpath_saves_references_locally(
mock_log, clean_git_repo):
"""Validate that refpath/hexsha branches are created in the local repo
following clone of the repo using a refpath during initial checkout.
"""
# Clone the openstack-helm repo and automatically checkout patch 34.
ref = 'refs/changes/15/536215/34'
repo_url = 'https://github.com/openstack/openstack-helm'
git_dir = git.git_handler(repo_url, ref)
_validate_git_clone(git_dir, fetched_ref=ref)
# Now checkout patch 34 again to ensure it's still there.
ref = 'refs/changes/15/536215/34'
git_dir = git.git_handler(git_dir, ref)
_validate_git_clone(git_dir, checked_out_ref=ref)
_assert_repo_url_was_cloned(mock_log, git_dir)
# Verify that passing in the hexsha variation of refpath
# 'refs/changes/15/536215/34' also works.
hexref = '276102a115dac3c0a6e91f9047d8b086bc8d2ff0'
git_dir = git.git_handler(git_dir, hexref)
_validate_git_clone(git_dir, checked_out_ref=hexref)
_assert_repo_url_was_cloned(mock_log, git_dir)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
@mock.patch.object(git, 'LOG', autospec=True)
def test_git_clone_checkout_hexsha_saves_references_locally(
mock_log, clean_git_repo):
"""Validate that refpath/hexsha branches are created in the local repo
following clone of the repo using a hexsha during initial checkout.
"""
# Clone the openstack-helm repo and automatically checkout patch using
# hexsha.
# NOTE(felipemonteiro): We have to use the commit ID (hexsha) corresponding
# to the last patch as that is what gets pushed to github. In this case,
# this corresponds to patch 'refs/changes/15/536215/35'.
ref = 'bf126f46b1c175a8038949a87dafb0a716e3b9b6'
repo_url = 'https://github.com/openstack/openstack-helm'
git_dir = git.git_handler(repo_url, ref)
_validate_git_clone(git_dir, fetched_ref=ref)
# Now checkout patch using hexsha again to ensure it's still there.
ref = 'bf126f46b1c175a8038949a87dafb0a716e3b9b6'
git_dir = git.git_handler(git_dir, ref)
_validate_git_clone(git_dir, checked_out_ref=ref)
_assert_repo_url_was_cloned(mock_log, git_dir)
# Verify that passing in the refpath variation of hexsha also works.
hexref = 'refs/changes/15/536215/35'
git_dir = git.git_handler(git_dir, hexref)
_validate_git_clone(git_dir, checked_out_ref=hexref)
_assert_repo_url_was_cloned(mock_log, git_dir)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
@mock.patch.object(git, 'LOG', autospec=True)
def test_git_clone_existing_directory_checks_out_next_local_ref(
mock_log, clean_git_repo):
"""Validate Git fetches the newer ref upstream that doesn't exist locally
in the cloned repo.
"""
# Clone the openstack-helm repo and automatically checkout patch 73.
ref = 'refs/changes/54/457754/73'
repo_url = 'https://github.com/openstack/openstack-helm'
git_dir = git.git_handler(repo_url, ref)
_validate_git_clone(git_dir, ref)
# Attempt to checkout patch 74 which requires a remote fetch even though
# the repo has already been cloned.
ref = 'refs/changes/54/457754/74'
git_dir = git.git_handler(git_dir, ref)
_validate_git_clone(git_dir, ref)
_assert_repo_url_was_cloned(mock_log, git_dir)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
@mock.patch.object(git, 'LOG', autospec=True)
def test_git_clone_delete_repo_and_reclone(mock_log, clean_git_repo):
"""Validate that cloning a repo, then deleting it, then recloning it works.
"""
# Clone the openstack-helm repo and automatically checkout patch 73.
ref = 'refs/changes/54/457754/73'
repo_url = 'https://github.com/openstack/openstack-helm'
first_git_dir = git.git_handler(repo_url, ref)
_validate_git_clone(first_git_dir, ref)
# Validate that the repo was cloned.
assert mock_log.debug.called
mock_log.debug.assert_any_call('Cloning [%s]', repo_url)
mock_log.debug.reset_mock()
# Delete the just-cloned repo.
shutil.rmtree(first_git_dir)
# Verify that checking out the same ref results in a re-clone.
second_git_dir = git.git_handler(repo_url, ref)
_validate_git_clone(second_git_dir, ref)
# Validate that the repo was cloned.
assert first_git_dir != second_git_dir
assert mock_log.debug.called
mock_log.debug.assert_any_call('Cloning [%s]', repo_url)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
@mock.patch.object(git, 'LOG', autospec=True)
def test_git_clone_clean_dirty_local_repo(mock_log, clean_git_repo):
"""Validate that a dirty repo is cleaned before a ref is checked out."""
ref = 'refs/changes/54/457754/73'
repo_url = 'https://github.com/openstack/openstack-helm'
git_dir = git.git_handler(repo_url, ref)
_validate_git_clone(git_dir, ref)
file_to_rename = os.path.join(git_dir, os.listdir(git_dir)[0])
os.rename(file_to_rename, file_to_rename + '-renamed')
git_dir = git.git_handler(git_dir, ref)
_validate_git_clone(git_dir, ref)
assert mock_log.warning.called
mock_log.warning.assert_any_call(
'The locally cloned repo_url=%s is dirty. Cleaning up untracked '
'files.', git_dir)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
@mock.patch.object(git, 'LOG', autospec=True)
def test_git_clone_existing_directory_raises_exc_for_invalid_ref(
mock_log, clean_git_repo):
"""Validate Git throws an error for an invalid ref when trying to checkout
a ref for an already-cloned repo.
"""
# Clone the openstack-helm repo and automatically checkout patch 73.
ref = 'refs/changes/54/457754/73'
repo_url = 'https://github.com/openstack/openstack-helm'
git_dir = git.git_handler(repo_url, ref)
_validate_git_clone(git_dir, ref)
# Attempt to checkout patch 9000 now that the repo already exists locally.
ref = 'refs/changes/54/457754/9000'
with pytest.raises(exceptions.GitException):
git_dir = git.git_handler(git_dir, ref)
_assert_repo_url_was_cloned(mock_log, git_dir)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
def test_git_clone_empty_url_raises_value_error(clean_git_repo):
url = ''
with pytest.raises(ValueError):
git.git_handler(url, ref='master')
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
def test_git_clone_invalid_url_type_raises_value_error(clean_git_repo):
url = 5
with pytest.raises(ValueError):
git.git_handler(url, ref='master')
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
def test_git_clone_invalid_local_repo_url_raises_notadirectory_error(
clean_git_repo):
url = False
with pytest.raises(NotADirectoryError):
git.git_handler(url, ref='master')
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
def test_git_clone_invalid_remote_url(clean_git_repo):
url = 'https://github.com/dummy/armada'
with pytest.raises(exceptions.GitException):
git.git_handler(url, ref='master')
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
def test_git_clone_invalid_remote_url_protocol(clean_git_repo):
url = 'ftp://foo.bar'
with pytest.raises(ValueError):
git.git_handler(url, ref='master')
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
def test_git_clone_fake_proxy(clean_git_repo):
url = 'https://github.com/openstack/airship-armada'
proxy_url = test_utils.rand_name(
'not.a.proxy.that.works.and.never.will', prefix='http://') + ":8080"
with pytest.raises(exceptions.GitProxyException):
git.git_handler(url, ref='master', proxy_server=proxy_url)
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
@mock.patch('os.path.exists', return_value=True, autospec=True)
def test_git_clone_ssh_auth_method_fails_auth(_, clean_git_repo):
fake_user = test_utils.rand_name('fake_user')
url = ('ssh://%s@review.openstack.org:29418/openstack/airship-armada' %
fake_user)
with pytest.raises(exceptions.GitAuthException):
git._try_git_clone(
url, ref='refs/changes/17/388517/5', auth_key='/home/user/.ssh/')
@pytest.mark.skipif(
not is_connected(), reason='git clone requires network connectivity.')
@mock.patch('os.path.exists', return_value=False, autospec=True)
def test_git_clone_ssh_auth_method_missing_ssh_key(_, clean_git_repo):
fake_user = test_utils.rand_name('fake_user')
url = ('ssh://%s@review.openstack.org:29418/openstack/airship-armada' %
fake_user)
with pytest.raises(exceptions.GitSSHException):
git.git_handler(
url, ref='refs/changes/17/388517/5', auth_key='/home/user/.ssh/')

View File

@ -47,6 +47,9 @@ def _gen_document(**kwargs):
def create_tmp_deployment_files(tmpdir): def create_tmp_deployment_files(tmpdir):
"""Fixture that creates a temporary directory structure.""" """Fixture that creates a temporary directory structure."""
sitenames = ['cicd', 'lab'] sitenames = ['cicd', 'lab']
# Used for ensuring the original global context is reset in memory
# following each test execution.
original_global_context = copy.deepcopy(config.GLOBAL_CONTEXT)
SITE_TEST_STRUCTURE = { SITE_TEST_STRUCTURE = {
'directories': { 'directories': {
@ -152,8 +155,5 @@ schema: pegleg/SiteDefinition/v1
yield yield
config.GLOBAL_CONTEXT = { # Restore the global context back to blank slate status.
'primary_repo': './', config.GLOBAL_CONTEXT = original_global_context
'aux_repos': [],
'site_path': 'site'
}

View File

@ -0,0 +1,39 @@
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2015 Hewlett-Packard Development Company, L.P.
# Copyright 2017 AT&T Intellectual Property.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import random
import uuid
def rand_name(name='', prefix='armada'):
"""Generate a random name that includes a random number
:param str name: The name that you want to include
:param str prefix: The prefix that you want to include
:return: a random name. The format is
'<prefix>-<name>-<random number>'.
(e.g. 'prefixfoo-namebar-154876201')
:rtype: string
"""
randbits = str(random.randint(1, 0x7fffffff))
rand_name = randbits
if name:
rand_name = name + '-' + rand_name
if prefix:
rand_name = prefix + '-' + rand_name
return rand_name

View File

@ -33,7 +33,10 @@ commands =
whitelist_externals = tox whitelist_externals = tox
[testenv:docs] [testenv:docs]
deps = -r{toxinidir}/doc/requirements.txt basepython = python3
deps =
-r{toxinidir}/src/bin/pegleg/requirements.txt
-r{toxinidir}/doc/requirements.txt
commands = commands =
rm -rf doc/build rm -rf doc/build
sphinx-build -b html doc/source doc/build -n -W -v sphinx-build -b html doc/source doc/build -n -W -v