# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import atexit import logging import os import re import shutil import tempfile import click from pegleg import config from pegleg.engine import exceptions from pegleg.engine import util __all__ = ('process_repositories', 'process_site_repository') __REPO_FOLDERS = {} _INVALID_FORMAT_MSG = ("The repository %s must be in the form of " "name=repoUrl[@revision]") LOG = logging.getLogger(__name__) @atexit.register def _clean_temp_folders(): global __REPO_FOLDERS for r in __REPO_FOLDERS.values(): shutil.rmtree(r, ignore_errors=True) def process_repositories(site_name, overwrite_existing=False): """Process and setup all repositories including ensuring we are at the right revision based on the site's own site-definition.yaml file. :param site_name: Site name for which to clone relevant repos. :param overwrite_existing: Whether to overwrite an existing directory """ # Only tracks extra repositories - not the site (primary) repository. extra_repos = [] site_repo = process_site_repository(overwrite_existing=overwrite_existing) # Retrieve extra repo data from site-definition.yaml files. site_data = util.definition.load_as_params( site_name, 'site_type', 'repositories', primary_repo_base=site_repo) site_def_repos = _get_and_validate_site_repositories(site_name, site_data) # Dict mapping repository names to associated URL/revision info for clone. repo_overrides = _process_repository_overrides(site_def_repos) if not site_def_repos: LOG.info('No repositories found in site-definition.yaml for site: %s. ' 'Defaulting to specified repository overrides.', site_name) site_def_repos = repo_overrides # Extract user/key that we will use for all repositories. repo_key = config.get_repo_key() repo_user = config.get_repo_username() for repo_alias in site_def_repos.keys(): if repo_alias == "site": LOG.warning("The primary site repository path must be specified " "via the -r flag. Ignoring the provided " "site-definition entry: %s", site_def_repos[repo_alias]) continue # Extract URL and revision, prioritizing overrides over the defaults in # the site-definition.yaml. if repo_alias in repo_overrides: repo_url_or_path = repo_overrides[repo_alias]['url'] repo_revision = repo_overrides[repo_alias]['revision'] else: repo_url_or_path = site_def_repos[repo_alias]['url'] repo_revision = site_def_repos[repo_alias]['revision'] repo_url_or_path = _format_url_with_repo_username(repo_url_or_path) LOG.info("Processing repository %s with url=%s, repo_key=%s, " "repo_username=%s, revision=%s", repo_alias, repo_url_or_path, repo_key, repo_user, repo_revision) temp_extra_repo = _process_repository( repo_url_or_path, repo_revision, overwrite_existing=overwrite_existing) extra_repos.append(temp_extra_repo) # Overwrite the site repo and extra repos in the config because further # processing will fail if they contain revision info in their paths. LOG.debug("Updating site_repo=%s extra_repo_list=%s in config", site_repo, extra_repos) config.set_site_repo(site_repo) config.set_extra_repo_list(extra_repos) def process_site_repository(update_config=False, overwrite_existing=False): """Process and setup site repository including ensuring we are at the right revision based on the site's own site-definition.yaml file. :param bool update_config: Whether to update Pegleg config with computed site repo path. :param overwrite_existing: Whether to overwrite an existing directory """ # Retrieve the main site repository and validate it. site_repo_or_path = config.get_site_repo() if not site_repo_or_path: raise ValueError("Site repository directory (%s) must be specified" % site_repo_or_path) repo_url_or_path, repo_revision = _extract_repo_url_and_revision( site_repo_or_path) config.set_site_rev(repo_revision) repo_url_or_path = _format_url_with_repo_username(repo_url_or_path) new_repo_path = _process_repository(repo_url_or_path, repo_revision, overwrite_existing=overwrite_existing) if update_config: # Overwrite the site repo in the config because further processing will # fail if they contain revision info in their paths. LOG.debug("Updating site_repo=%s in config", new_repo_path) config.set_site_repo(new_repo_path) return new_repo_path def _process_repository(repo_url_or_path, repo_revision, overwrite_existing=False): """Process a repository located at ``repo_url_or_path``. :param str repo_url_or_path: Path to local repo or URL of remote URL. :param str repo_revision: branch, commit or ref in the repo to checkout. :param overwrite_existing: Whether to overwrite an existing directory """ global __REPO_FOLDERS if os.path.exists(repo_url_or_path) and not overwrite_existing: repo_name = util.git.repo_name(repo_url_or_path) parent_temp_path = tempfile.mkdtemp() __REPO_FOLDERS.setdefault(repo_name, parent_temp_path) new_temp_path = os.path.join(parent_temp_path, repo_name) norm_path, sub_path = util.git.normalize_repo_path(repo_url_or_path) shutil.copytree(src=norm_path, dst=new_temp_path, symlinks=True) __REPO_FOLDERS.setdefault(repo_name, new_temp_path) git_repo_path = _process_site_repository(new_temp_path, repo_revision) return os.path.join(git_repo_path, sub_path) else: repo_url, sub_path = util.git.normalize_repo_path(repo_url_or_path) git_repo_path = _process_site_repository(repo_url, repo_revision) return os.path.join(git_repo_path, sub_path) def _process_site_repository(repo_url_or_path, repo_revision): """Process the primary or site repository located at ``repo_url_or_path``. Also validate that the provided ``repo_url_or_path`` is a valid Git repository. If ``repo_url_or_path`` doesn't already exist, clone it. If it does, extract the appropriate revision and check it out. :param repo_url_or_path: Repo path or URL and associated auth information. If URL, examples include: * ssh://REPO_USERNAME@:29418/global-manifests.git@ * https:///global-manifests.git@ * http:///global-manifests.git@ * @ * same values as above without @ :param str repo_revision: branch, commit or ref in the repo to checkout. """ repo_alias = 'site' # We are processing the site repo necessarily. repo_key = config.get_repo_key() repo_user = config.get_repo_username() LOG.info("Processing repository %s with url=%s, repo_key=%s, " "repo_username=%s, revision=%s", repo_alias, repo_url_or_path, repo_key, repo_user, repo_revision) return _handle_repository( repo_url_or_path, ref=repo_revision, auth_key=repo_key) def _get_and_validate_site_repositories(site_name, site_data): """Validate that repositories entry exists in ``site_data``.""" if 'repositories' not in site_data: LOG.info("The repository for site_name: %s does not contain a " "site-definition.yaml with a 'repositories' key. Ensure " "your repository is self-contained and doesn't require " "extra repositories for correct rendering.", site_name) return site_data.get('repositories', {}) def _process_repository_overrides(site_def_repos): """Process specified repository overrides via the CLI ``-e`` flag. This will resolve ``-e`` site CLI arguments and override the corresponding values in the relevant :file:`site-definition.yaml`, if applicable. For example, given CLI override of:: -e global=/opt/global@foo And site-definition.yaml ``repositories`` value of:: repositories: global: revision: bar url: /opt/global Then the resulting dictionary that is returned will be:: {"global": {"url": "/opt/global", "revision": "foo"}} :param site_def_repos: Dictionary of ``repositories`` field included in relevant :file:`site-definition.yaml`. :returns: Dictionary with above format. """ # Extra repositories to process. provided_repo_overrides = config.get_extra_repo_overrides() # Map repository names to the associated URL/revision for cloning. repo_overrides = {} for repo_override in provided_repo_overrides: # break apart global=repoUrl try: repo_alias, repo_url_or_path = repo_override.split('=', 1) except ValueError: # TODO(felipemonteiro): Use internal exceptions for this. raise click.ClickException(_INVALID_FORMAT_MSG % repo_override) if repo_alias == "site": LOG.warning("The primary site repository path must be specified " "via the -r flag. Ignoring the provided override: %s", repo_override) continue if repo_alias not in site_def_repos: # If we are overriding a value that doesn't exist in the # site-definition.yaml make a note of it in case the override # is something bogus, but we won't make this a hard requirement, # so just log the discrepancy. LOG.debug("Repo override: %s not found under `repositories` for " "site-definition.yaml. Site def repositories: %s", repo_override, ", ".join(site_def_repos.keys())) repo_url, revision = _extract_repo_url_and_revision(repo_url_or_path) # store what we've learned repo_overrides.setdefault(repo_alias, {}) repo_overrides[repo_alias]['url'] = repo_url repo_overrides[repo_alias]['revision'] = revision return repo_overrides def _extract_repo_url_and_revision(repo_url_or_path): """Break up repository path/url into the repo URL and revision. :param repo_url_or_path: Repo URL and associated auth information. E.g.: * ssh://REPO_USERNAME@:29418/global-manifests.git@ * https:///global-manifests.git@ * http:///global-manifests.git@ * @ * same values as above without @ """ ssh_username_pattern = re.compile('ssh:\/\/.+@.+\/.+') def has_revision(repo_url_or_path): if repo_url_or_path.lower().startswith('ssh'): if ssh_username_pattern.match(repo_url_or_path): return repo_url_or_path.count('@') == 2 return repo_url_or_path.count('@') == 1 else: return '@' in repo_url_or_path # they've forced a revision using @revision - careful not to confuse # this with auth revision = None try: if has_revision(repo_url_or_path): repo_url_or_path, revision = repo_url_or_path.rsplit('@', 1) revision = revision.rstrip('/') if revision.endswith(".git"): revision = revision[:-4] if repo_url_or_path.endswith(".git"): repo_url_or_path = repo_url_or_path[:-4] else: repo_url_or_path = repo_url_or_path except Exception: # TODO(felipemonteiro): Use internal exceptions for this. raise click.ClickException(_INVALID_FORMAT_MSG % repo_url_or_path) return repo_url_or_path, revision def _format_url_with_repo_username(repo_url_or_path): # If a repo user is provided, do the necessary replacements. repo_user = config.get_repo_username() if repo_user: if "REPO_USERNAME" not in repo_url_or_path: LOG.warning( "A repository username was specified but no REPO_USERNAME " "string found in repository url %s", repo_url_or_path) else: repo_url_or_path = repo_url_or_path.replace( 'REPO_USERNAME', repo_user) return repo_url_or_path def _handle_repository(repo_url_or_path, *args, **kwargs): """Clone remote remote (if ``repo_url_or_path`` is a remote URL) and checkout specified reference . """ # Retrieve the clone path where the repo will be cloned clone_path = config.get_clone_path() try: return util.git.git_handler( repo_url_or_path, clone_path=clone_path, *args, **kwargs) except exceptions.GitException as e: raise click.ClickException(e) except Exception as e: LOG.exception('Unknown exception was raised during git clone/checkout:' ' %s', e) # TODO(felipemonteiro): Use internal exceptions for this. raise click.ClickException(e)