diff --git a/Dockerfile b/Dockerfile index de805667..72deb33f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM python:3.5 MAINTAINER Armada Team @@ -6,33 +6,20 @@ ENV DEBIAN_FRONTEND noninteractive ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 -COPY . /armada +COPY requirements.txt /tmp/ +RUN pip3 install -r /tmp/requirements.txt +COPY . /armada RUN apt-get update && \ - apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ netbase \ - python3-pip && \ - apt-get install -y \ - build-essential \ curl \ - git \ - python3-minimal \ - python3-setuptools \ - python3-dev && \ + git && \ useradd -u 1000 -g users -d /armada armada && \ chown -R armada:users /armada && \ mv /armada/etc/armada /etc/ && \ - \ cd /armada && \ - pip3 install --upgrade pip && \ - pip3 install -r requirements.txt && \ python3 setup.py install && \ - \ - apt-get purge --auto-remove -y \ - build-essential \ - curl && \ - apt-get clean -y && \ rm -rf \ /root/.cache \ /var/lib/apt/lists/* diff --git a/armada/api/__init__.py b/armada/api/__init__.py index 3d4c7ef1..db6df8cb 100644 --- a/armada/api/__init__.py +++ b/armada/api/__init__.py @@ -59,6 +59,24 @@ class BaseResource(object): raise Exception( "%s: Invalid YAML in body: %s" % (req.path, jex)) + def req_json(self, req): + if req.content_length is None or req.content_length == 0: + return None + + raw_body = req.stream.read(req.content_length or 0) + + if raw_body is None: + return None + + try: + return json.loads(raw_body.decode()) + except json.JSONDecodeError as jex: + self.error( + req.context, + "Invalid JSON in request: %s" % str(jex)) + raise Exception( + "%s: Invalid JSON in body: %s" % (req.path, jex)) + def return_error(self, resp, status_code, message="", retry=False): resp.body = json.dumps({ 'type': 'error', diff --git a/armada/api/controller/armada.py b/armada/api/controller/armada.py index ad298db4..639470d8 100644 --- a/armada/api/controller/armada.py +++ b/armada/api/controller/armada.py @@ -13,12 +13,15 @@ # limitations under the License. import json +import yaml import falcon from armada import api from armada.common import policy from armada.handlers.armada import Armada +from armada.handlers.document import ReferenceResolver +from armada.handlers.override import Override class Apply(api.BaseResource): @@ -30,17 +33,45 @@ class Apply(api.BaseResource): try: # Load data from request and get options + if req.content_type == 'application/x-yaml': + data = list(self.req_yaml(req)) + if type(data[0]) is list: + documents = list(data[0]) + else: + documents = data + elif req.content_type == 'application/json': + self.logger.debug("Applying manifest based on reference.") + req_body = self.req_json(req) + doc_ref = req_body.get('hrefs', None) - data = list(self.req_yaml(req)) + if not doc_ref: + self.logger.info("Request did not contain 'hrefs'.") + resp.status = falcon.HTTP_400 + return - if type(data[0]) is list: - data = list(data[0]) + data = ReferenceResolver.resolve_reference(doc_ref) + documents = list() + for d in data: + documents.extend(list(yaml.safe_load_all(d.decode()))) + + if req_body.get('overrides', None): + overrides = Override(documents, + overrides=req_body.get('overrides')) + documents = overrides.update_manifests() + else: + self.error(req.context, "Unknown content-type %s" + % req.content_type) + self.return_error( + resp, + falcon.HTTP_415, + message="Request must be in application/x-yaml" + "or application/json") opts = req.params # Encode filename armada = Armada( - data, + documents, disable_update_pre=req.get_param_as_bool( 'disable_update_pre'), disable_update_post=req.get_param_as_bool( diff --git a/armada/api/controller/validation.py b/armada/api/controller/validation.py index 7e643296..1cb7deaf 100644 --- a/armada/api/controller/validation.py +++ b/armada/api/controller/validation.py @@ -13,12 +13,13 @@ # limitations under the License. import json - import falcon +import yaml from armada import api from armada.common import policy from armada.utils.lint import validate_armada_documents +from armada.handlers.document import ReferenceResolver class Validate(api.BaseResource): @@ -29,19 +30,58 @@ class Validate(api.BaseResource): @policy.enforce('armada:validate_manifest') def on_post(self, req, resp): try: - manifest = self.req_yaml(req) - documents = list(manifest) + if req.content_type == 'application/json': + self.logger.debug("Validating manifest based on reference.") + json_body = self.req_json(req) + if json_body.get('href', None): + self.logger.debug("Validating manifest from reference %s." + % json_body.get('href')) + data = ReferenceResolver.resolve_reference( + json_body.get('href')) + documents = list() + for d in data: + documents.extend(list(yaml.safe_load_all(d.decode()))) + else: + resp.status = falcon.HTTP_400 + return + else: + manifest = self.req_yaml(req) + documents = list(manifest) - message = { - 'valid': validate_armada_documents(documents) + self.logger.debug("Validating set of %d documents." + % len(documents)) + + result = validate_armada_documents(documents) + + resp.content_type = 'application/json' + resp_body = { + 'kind': 'Status', + 'apiVersion': 'v1.0', + 'metadata': {}, + 'reason': 'Validation', + 'details': { + 'errorCount': 0, + 'messageList': [] + }, } - resp.status = falcon.HTTP_200 - resp.body = json.dumps(message) - resp.content_type = 'application/json' + if result: + resp.status = falcon.HTTP_200 + resp_body['status'] = 'Success' + resp_body['message'] = 'Armada validations succeeded' + resp_body['code'] = 200 + else: + resp.status = falcon.HTTP_400 + resp_body['status'] = 'Failure' + resp_body['message'] = 'Armada validations failed' + resp_body['code'] = 400 + resp_body['details']['errorCount'] = 1 + resp_body['details']['messageList'].\ + append(dict(message='Validation failed.', error=True)) - except Exception: + resp.body = json.dumps(resp_body) + except Exception as ex: err_message = 'Failed to validate Armada Manifest' - self.error(req.context, err_message) + self.logger.error(err_message, exc_info=ex) self.return_error( resp, falcon.HTTP_400, message=err_message) diff --git a/armada/api/server.py b/armada/api/server.py index 341297ff..9883a0ee 100644 --- a/armada/api/server.py +++ b/armada/api/server.py @@ -15,6 +15,7 @@ import falcon from oslo_config import cfg from oslo_policy import policy +from oslo_log import log as logging from armada import conf from armada.api import ArmadaRequest @@ -52,6 +53,9 @@ def create(enable_middleware=CONF.middleware): else: api = falcon.API(request_type=ArmadaRequest) + logging.set_defaults(default_log_levels=CONF.default_log_levels) + logging.setup(CONF, 'armada') + # Configure API routing url_routes_v1 = ( ('health', Health()), @@ -61,6 +65,7 @@ def create(enable_middleware=CONF.middleware): ('tests', Tests()), ('test/{release}', Test()), ('validate', Validate()), + ('validatedesign', Validate()), ) for route, service in url_routes_v1: diff --git a/armada/cli/apply.py b/armada/cli/apply.py index dc96f313..90f37b74 100644 --- a/armada/cli/apply.py +++ b/armada/cli/apply.py @@ -18,7 +18,9 @@ import click from oslo_config import cfg from armada.cli import CliAction +from armada.exceptions.source_exceptions import InvalidPathException from armada.handlers.armada import Armada +from armada.handlers.document import ReferenceResolver CONF = cfg.CONF @@ -64,7 +66,7 @@ SHORT_DESC = "command install manifest charts" @apply.command(name='apply', help=DESC, short_help=SHORT_DESC) -@click.argument('filename') +@click.argument('locations', nargs=-1) @click.option('--api', help="Contacts service endpoint", is_flag=True) @click.option( '--disable-update-post', help="run charts without install", is_flag=True) @@ -78,66 +80,35 @@ SHORT_DESC = "command install manifest charts" @click.option( '--tiller-port', help="Tiller host port", type=int, default=44134) @click.option( - '--timeout', help="specifies time to wait for charts", type=int, + '--timeout', + help="specifies time to wait for charts", + type=int, default=3600) @click.option('--values', '-f', multiple=True, type=str, default=[]) -@click.option( - '--wait', help="wait until all charts deployed", is_flag=True) +@click.option('--wait', help="wait until all charts deployed", is_flag=True) @click.option( '--debug/--no-debug', help='Enable or disable debugging', default=False) @click.pass_context -def apply_create(ctx, - filename, - api, - disable_update_post, - disable_update_pre, - dry_run, - enable_chart_cleanup, - set, - tiller_host, - tiller_port, - timeout, - values, - wait, - debug): +def apply_create(ctx, locations, api, disable_update_post, disable_update_pre, + dry_run, enable_chart_cleanup, set, tiller_host, tiller_port, + timeout, values, wait, debug): if debug: CONF.debug = debug - ApplyManifest( - ctx, - filename, - api, - disable_update_post, - disable_update_pre, - dry_run, - enable_chart_cleanup, - set, - tiller_host, - tiller_port, - timeout, - values, - wait).invoke() + ApplyManifest(ctx, locations, api, disable_update_post, disable_update_pre, + dry_run, enable_chart_cleanup, set, tiller_host, tiller_port, + timeout, values, wait).invoke() class ApplyManifest(CliAction): - def __init__(self, - ctx, - filename, - api, - disable_update_post, - disable_update_pre, - dry_run, - enable_chart_cleanup, - set, - tiller_host, - tiller_port, - timeout, - values, - wait): + def __init__(self, ctx, locations, api, disable_update_post, + disable_update_pre, dry_run, enable_chart_cleanup, set, + tiller_host, tiller_port, timeout, values, wait): super(ApplyManifest, self).__init__() self.ctx = ctx - self.filename = filename + # Filename can also be a URL reference + self.locations = locations self.api = api self.disable_update_post = disable_update_post self.disable_update_pre = disable_update_pre @@ -153,8 +124,7 @@ class ApplyManifest(CliAction): def output(self, resp): for result in resp: if not resp[result] and not result == 'diff': - self.logger.info( - 'Did not performed chart %s(s)', result) + self.logger.info('Did not performed chart %s(s)', result) elif result == 'diff' and not resp[result]: self.logger.info('No Relase changes detected') @@ -167,25 +137,32 @@ class ApplyManifest(CliAction): self.logger.info(ch) def invoke(self): - if not self.ctx.obj.get('api', False): - with open(self.filename) as f: - armada = Armada( - list(yaml.safe_load_all(f.read())), - self.disable_update_pre, - self.disable_update_post, - self.enable_chart_cleanup, - self.dry_run, - self.set, - self.wait, - self.timeout, - self.tiller_host, - self.tiller_port, - self.values) + try: + doc_data = ReferenceResolver.resolve_reference(self.locations) + documents = list() + for d in doc_data: + documents.extend(list(yaml.safe_load_all(d.decode()))) + except InvalidPathException as ex: + self.logger.error(str(ex)) + return + except yaml.YAMLError as yex: + self.logger.error("Invalid YAML found: %s" % str(yex)) + return - resp = armada.sync() - self.output(resp) + armada = Armada( + documents, self.disable_update_pre, self.disable_update_post, + self.enable_chart_cleanup, self.dry_run, self.set, self.wait, + self.timeout, self.tiller_host, self.tiller_port, self.values) + + resp = armada.sync() + self.output(resp) else: + if len(self.values) > 0: + self.logger.error( + "Cannot specify local values files when using the API.") + return + query = { 'disable_update_post': self.disable_update_post, 'disable_update_pre': self.disable_update_pre, @@ -199,8 +176,6 @@ class ApplyManifest(CliAction): client = self.ctx.obj.get('CLIENT') - with open(self.filename, 'r') as f: - resp = client.post_apply( - manifest=f.read(), values=self.values, set=self.set, - query=query) - self.output(resp.get('message')) + resp = client.post_apply( + manifest_ref=self.locations, set=self.set, query=query) + self.output(resp.get('message')) diff --git a/armada/cli/validate.py b/armada/cli/validate.py index c2c67be6..1e92c2e8 100644 --- a/armada/cli/validate.py +++ b/armada/cli/validate.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import click import yaml @@ -20,6 +19,7 @@ from armada.cli import CliAction from armada.utils.lint import validate_armada_documents from armada.utils.lint import validate_armada_object from armada.handlers.manifest import Manifest +from armada.handlers.document import ReferenceResolver @click.group() @@ -42,38 +42,47 @@ SHORT_DESC = "command validates Armada Manifest" @validate.command(name='validate', help=DESC, short_help=SHORT_DESC) -@click.argument('filename') +@click.argument('locations', nargs=-1) @click.pass_context -def validate_manifest(ctx, filename): - ValidateManifest(ctx, filename).invoke() +def validate_manifest(ctx, locations): + ValidateManifest(ctx, locations).invoke() class ValidateManifest(CliAction): - - def __init__(self, ctx, filename): + def __init__(self, ctx, locations): super(ValidateManifest, self).__init__() self.ctx = ctx - self.filename = filename + self.locations = locations def invoke(self): if not self.ctx.obj.get('api', False): - documents = yaml.safe_load_all(open(self.filename).read()) + doc_data = ReferenceResolver.resolve_reference(self.locations) + documents = list() + for d in doc_data: + documents.extend(list(yaml.safe_load_all(d.decode()))) + manifest_obj = Manifest(documents).get_manifest() obj_check = validate_armada_object(manifest_obj) doc_check = validate_armada_documents(documents) try: if doc_check and obj_check: - self.logger.info( - 'Successfully validated: %s', self.filename) + self.logger.info('Successfully validated: %s', + self.locations) except Exception: - raise Exception('Failed to validate: %s', self.filename) + raise Exception('Failed to validate: %s', self.locations) else: + if len(self.locations) > 1: + self.logger.error( + "Cannot specify multiple locations " + "when using validate API." + ) + return + client = self.ctx.obj.get('CLIENT') - with open(self.filename, 'r') as f: - resp = client.post_validate(f.read()) - if resp.get('valid', False): - self.logger.info( - 'Successfully validated: %s', self.filename) - else: - self.logger.error("Failed to validate: %s", self.filename) + resp = client.post_validate(self.locations[0]) + + if resp.get('code') == 200: + self.logger.info('Successfully validated: %s', self.locations) + else: + self.logger.error("Failed to validate: %s", self.locations) diff --git a/armada/common/client.py b/armada/common/client.py index 9853219c..21217b15 100644 --- a/armada/common/client.py +++ b/armada/common/client.py @@ -27,7 +27,6 @@ API_VERSION = 'v{}/{}' class ArmadaClient(object): - def __init__(self, session): self.session = session @@ -55,22 +54,72 @@ class ArmadaClient(object): def post_validate(self, manifest=None): endpoint = self._set_endpoint('1.0', 'validate') - resp = self.session.post(endpoint, body=manifest) + # TODO(sh8121att) Look to update the UCP convention to + # allow a list of hrefs + req_body = {'href': manifest} + + resp = self.session.post( + endpoint, + data=req_body, + headers={ + 'content-type': 'application/json' + }) self._check_response(resp) return resp.json() - def post_apply(self, manifest=None, values=None, set=None, query=None): + def post_apply(self, + manifest=None, + manifest_ref=None, + values=None, + set=None, + query=None): + """Call the Armada API to apply a manifest. - if values or set: - document = list(yaml.safe_load_all(manifest)) - override = Override( - document, overrides=set, values=values).update_manifests() - manifest = yaml.dump(override) + If manifest is not None, then the request body will be a fully + rendered set of YAML documents including overrides and + values-files application. + If manifest is None and manifest_ref is not, then the request + body will be a JSON structure providing a list of references + to Armada manifest documents and a list of overrides. Local + values files are not supported when using the API with references. + + :param manifest: string of YAML formatted Armada manifests + :param manifest_ref: valid file paths or URIs referring to Armada + manifests + :param values: list of local files containing values.yaml overrides + :param set: list of single-value overrides + :param query: explicit query string parameters + """ endpoint = self._set_endpoint('1.0', 'apply') - resp = self.session.post(endpoint, body=manifest, query=query) + + if manifest: + if values or set: + document = list(yaml.safe_load_all(manifest)) + override = Override( + document, overrides=set, values=values).update_manifests() + manifest = yaml.dump(override) + resp = self.session.post( + endpoint, + body=manifest, + query=query, + headers={ + 'content-type': 'application/x-yaml' + }) + elif manifest_ref: + req_body = { + 'hrefs': manifest_ref, + 'overrides': set or [], + } + resp = self.session.post( + endpoint, + data=req_body, + query=query, + headers={ + 'content-type': 'application/json' + }) self._check_response(resp) @@ -100,8 +149,7 @@ class ArmadaClient(object): "Unauthorized access to %s, include valid token.".format( resp.url)) elif resp.status_code == 403: - raise err.ClientForbiddenError( - "Forbidden access to %s" % resp.url) + raise err.ClientForbiddenError("Forbidden access to %s" % resp.url) elif not resp.ok: - raise err.ClientError( - "Error - received %d: %s" % (resp.status_code, resp.text)) + raise err.ClientError("Error - received %d: %s" % + (resp.status_code, resp.text)) diff --git a/armada/common/session.py b/armada/common/session.py index 558efd33..302e4fbb 100644 --- a/armada/common/session.py +++ b/armada/common/session.py @@ -55,22 +55,23 @@ class ArmadaSession(object): self.logger = LOG # TODO Add keystone authentication to produce a token for this session - def get(self, endpoint, query=None): + def get(self, endpoint, query=None, headers=None): """ Send a GET request to armada. :param string endpoint: URL string following hostname and API prefix :param dict query: A dict of k, v pairs to add to the query string + :param headers: Dictionary of HTTP headers to include in request :return: A requests.Response object """ api_url = '{}{}'.format(self.base_url, endpoint) resp = self._session.get( - api_url, params=query, timeout=3600) + api_url, params=query, headers=headers, timeout=3600) return resp - def post(self, endpoint, query=None, body=None, data=None): + def post(self, endpoint, query=None, body=None, data=None, headers=None): """ Send a POST request to armada. If both body and data are specified, body will will be used. @@ -79,6 +80,7 @@ class ArmadaSession(object): :param dict query: dict of k, v parameters to add to the query string :param string body: string to use as the request body. :param data: Something json.dumps(s) can serialize. + :param headers: Dictionary of HTTP headers to include in request :return: A requests.Response object """ api_url = '{}{}'.format(self.base_url, endpoint) @@ -86,11 +88,17 @@ class ArmadaSession(object): self.logger.debug("Sending POST with armada_client session") if body is not None: self.logger.debug("Sending POST with explicit body: \n%s" % body) - resp = self._session.post( - api_url, params=query, data=body, timeout=3600) + resp = self._session.post(api_url, + params=query, + data=body, + headers=headers, + timeout=3600) else: self.logger.debug("Sending POST with JSON body: \n%s" % str(data)) - resp = self._session.post( - api_url, params=query, json=data, timeout=3600) + resp = self._session.post(api_url, + params=query, + json=data, + headers=headers, + timeout=3600) return resp diff --git a/armada/conf/__init__.py b/armada/conf/__init__.py index 7ff1256c..dc55175d 100644 --- a/armada/conf/__init__.py +++ b/armada/conf/__init__.py @@ -24,6 +24,9 @@ CONF = cfg.CONF CONFIG_FILES = ['api-paste.ini', 'armada.conf'] +# Load oslo_log options prior to file/CLI parsing +log.register_options(CONF) + def _get_config_files(env=None): if env is None: diff --git a/armada/handlers/document.py b/armada/handlers/document.py new file mode 100644 index 00000000..a546c090 --- /dev/null +++ b/armada/handlers/document.py @@ -0,0 +1,140 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for resolving design references.""" + +import urllib.parse +import re +import requests + +from oslo_log import log as logging + +from armada.exceptions.source_exceptions import InvalidPathException +from armada.utils.keystone import KeystoneUtils + +LOG = logging.getLogger(__name__) + + +class ReferenceResolver(object): + """Class for handling different data references to resolve them data.""" + + @classmethod + def resolve_reference(cls, design_ref): + """Resolve a reference to a design document. + + Locate a schema handler based on the URI scheme of the data reference + and use that handler to get the data referenced. + + :param design_ref: A list of URI-formatted reference to a data entity + :returns: A list of byte arrays + """ + data = [] + if isinstance(design_ref, str): + design_ref = [design_ref] + + for l in design_ref: + try: + LOG.debug("Resolving reference %s." % l) + design_uri = urllib.parse.urlparse(l) + + # when scheme is a empty string assume it is a local + # file path + if design_uri.scheme == '': + handler = cls.scheme_handlers.get('file') + else: + handler = cls.scheme_handlers.get(design_uri.scheme, None) + + if handler is None: + raise InvalidPathException( + "Invalid reference scheme %s: no handler." % + design_uri.scheme) + else: + # Have to do a little magic to call the classmethod + # as a pointer + data.append(handler.__get__(None, cls)(design_uri)) + except ValueError: + raise InvalidPathException( + "Cannot resolve design reference %s: unable " + "to parse as valid URI." + % l) + + return data + + @classmethod + def resolve_reference_http(cls, design_uri): + """Retrieve design documents from http/https endpoints. + + Return a byte array of the response content. Support + unsecured or basic auth + + :param design_uri: Tuple as returned by urllib.parse + for the design reference + """ + if design_uri.username is not None and design_uri.password is not None: + response = requests.get( + design_uri.geturl(), + auth=(design_uri.username, design_uri.password), + timeout=30) + else: + response = requests.get(design_uri.geturl(), timeout=30) + if response.status_code >= 400: + raise InvalidPathException( + "Error received for HTTP reference: %d" + % response.status_code) + + return response.content + + @classmethod + def resolve_reference_file(cls, design_uri): + """Retrieve design documents from local file endpoints. + + Return a byte array of the file contents + + :param design_uri: Tuple as returned by urllib.parse for the design + reference + """ + if design_uri.path != '': + with open(design_uri.path, 'rb') as f: + doc = f.read() + return doc + + @classmethod + def resolve_reference_ucp(cls, design_uri): + """Retrieve artifacts from a UCP service endpoint. + + Return a byte array of the response content. Assumes Keystone + authentication required. + + :param design_uri: Tuple as returned by urllib.parse for the design + reference + """ + ks_sess = KeystoneUtils.get_session() + (new_scheme, foo) = re.subn('^[^+]+\+', '', design_uri.scheme) + url = urllib.parse.urlunparse( + (new_scheme, design_uri.netloc, design_uri.path, design_uri.params, + design_uri.query, design_uri.fragment)) + LOG.debug("Calling Keystone session for url %s" % str(url)) + resp = ks_sess.get(url) + if resp.status_code >= 400: + raise InvalidPathException( + "Received error code for reference %s: %s - %s" % + (url, str(resp.status_code), resp.text)) + return resp.content + + scheme_handlers = { + 'http': resolve_reference_http, + 'file': resolve_reference_file, + 'https': resolve_reference_http, + 'deckhand+http': resolve_reference_ucp, + 'promenade+http': resolve_reference_ucp, + } diff --git a/armada/handlers/override.py b/armada/handlers/override.py index 3fd35d30..5bffbe85 100644 --- a/armada/handlers/override.py +++ b/armada/handlers/override.py @@ -31,7 +31,6 @@ class Override(object): ''' Retrieve yaml file as a dictionary. ''' - try: with open(doc) as f: return list(yaml.safe_load_all(f.read())) diff --git a/armada/tests/unit/api/test_armada_controller.py b/armada/tests/unit/api/test_armada_controller.py index 8d0473a8..6efa743a 100644 --- a/armada/tests/unit/api/test_armada_controller.py +++ b/armada/tests/unit/api/test_armada_controller.py @@ -17,7 +17,7 @@ import mock from oslo_config import cfg -from armada.handlers import armada +from armada.api.controller import armada as armada_api from armada.tests.unit.api import base CONF = cfg.CONF @@ -25,11 +25,9 @@ CONF = cfg.CONF class ArmadaControllerTest(base.BaseControllerTest): - @mock.patch.object(armada, 'lint') - @mock.patch.object(armada, 'Manifest') - @mock.patch.object(armada, 'Tiller') - def test_armada_apply_resource(self, mock_tiller, mock_manifest, - mock_lint): + @mock.patch.object(armada_api, 'Armada') + @mock.patch.object(armada_api, 'ReferenceResolver') + def test_armada_apply_resource(self, mock_resolver, mock_armada): """Tests the POST /api/v1.0/apply endpoint.""" rules = {'armada:create_endpoints': '@'} self.policy.set_rules(rules) @@ -42,17 +40,62 @@ class ArmadaControllerTest(base.BaseControllerTest): 'dry_run': 'false', 'wait': 'false', 'timeout': '100'} - payload = {'file': '', 'options': options} + + armada_options = { + 'disable_update_pre': False, + 'disable_update_post': False, + 'enable_chart_cleanup': False, + 'dry_run': False, + 'wait': False, + 'timeout': 100, + 'tiller_host': None, + 'tiller_port': 44134, + } + + payload_url = 'http://foo.com/test.yaml' + payload = {'hrefs': [payload_url]} body = json.dumps(payload) expected = {'message': {'diff': [], 'install': [], 'upgrade': []}} - result = self.app.simulate_post(path='/api/v1.0/apply', body=body) + mock_resolver.resolve_reference.return_value = \ + [b"---\nfoo: bar"] + + mock_armada.return_value.sync.return_value = \ + {'diff': [], 'install': [], 'upgrade': []} + + result = self.app.simulate_post(path='/api/v1.0/apply', + body=body, + headers={ + 'Content-Type': 'application/json' + }, + params=options) self.assertEqual(result.json, expected) self.assertEqual('application/json', result.headers['content-type']) - mock_tiller.assert_called_once_with(tiller_host=None, - tiller_port=44134) - mock_manifest.assert_called_once_with([payload]) - mock_lint.validate_armada_documents.assert_called_once_with([payload]) - fake_manifest = mock_manifest.return_value.get_manifest.return_value - mock_lint.validate_armada_object.assert_called_once_with(fake_manifest) + mock_resolver.resolve_reference.assert_called_with([payload_url]) + mock_armada.assert_called_with([{'foo': 'bar'}], **armada_options) + mock_armada.return_value.sync.assert_called() + + def test_armada_apply_no_href(self): + """Tests /api/v1.0/apply returns 400 when hrefs list is empty.""" + rules = {'armada:create_endpoints': '@'} + self.policy.set_rules(rules) + + options = {'debug': 'true', + 'disable_update_pre': 'false', + 'disable_update_post': 'false', + 'enable_chart_cleanup': 'false', + 'skip_pre_flight': 'false', + 'dry_run': 'false', + 'wait': 'false', + 'timeout': '100'} + payload = {'hrefs': []} + body = json.dumps(payload) + + result = self.app.simulate_post(path='/api/v1.0/apply', + body=body, + headers={ + 'Content-Type': 'application/json' + }, + params=options) + self.assertEqual(result.status_code, 400) diff --git a/armada/tests/unit/utils/test_lint.py b/armada/tests/unit/utils/test_lint.py index 18633c04..e54424cb 100644 --- a/armada/tests/unit/utils/test_lint.py +++ b/armada/tests/unit/utils/test_lint.py @@ -65,7 +65,7 @@ class LintTestCase(unittest.TestCase): schema: metadata/Document/v1 name: example-manifest data: - chart_groups: + chart_groups: - example-group """ document = yaml.safe_load_all(template_manifest) @@ -152,3 +152,17 @@ class LintTestCase(unittest.TestCase): document = yaml.safe_load_all(template_manifest) with self.assertRaises(Exception): lint.validate_chart_document(document) + + def test_lint_validate_manifest_url(self): + value = 'url' + assert lint.validate_manifest_url(value) is False + value = 'https://raw.githubusercontent.com/att-comdev/' \ + 'armada/master/examples/simple.yaml' + assert lint.validate_manifest_url(value) is True + + def test_lint_validate_manifest_filepath(self): + value = 'filepath' + assert lint.validate_manifest_filepath(value) is False + value = '{}/templates/valid_armada_document.yaml'.format( + self.basepath) + assert lint.validate_manifest_filepath(value) is True diff --git a/armada/utils/keystone.py b/armada/utils/keystone.py new file mode 100644 index 00000000..13dda4a5 --- /dev/null +++ b/armada/utils/keystone.py @@ -0,0 +1,64 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utility functions for accessing Openstack Keystone.""" + +import os + +from keystoneauth1.identity import v3 +from keystoneauth1 import session +from oslo_config import cfg + + +CONF = cfg.CONF + + +class KeystoneUtils(object): + """Utility methods for using Keystone.""" + + @staticmethod + def get_session(): + """Get an initialized keystone session. + + Authentication is based on the keystone_authtoken + section of the config file primarily. If that fails + then attempt to create a session from environmental + variables. This is for cases of the CLI needing + a token. + """ + auth_info = dict() + auth_fields = ['auth_url', 'username', 'password', 'project_id', + 'user_domain_name'] + try: + for f in auth_fields: + auth_info[f] = getattr(CONF.keystone_authtoken, f) + auth = v3.Password(**auth_info) + ks_session = session.Session(auth=auth) + # Test the session + ks_session.get_auth_headers() + except Exception: # nosec this isn't a security issue + pass + else: + return ks_session + + try: + for f in auth_fields: + auth_info[f] = os.environ.get('os_{}'.format(f).upper()) + auth = v3.Password(**auth_info) + ks_session = session.Session(auth=auth) + # Test the session + ks_session.get_auth_headers() + except Exception: + raise Exception('Missing credential information for Keystone.') + + return ks_session diff --git a/armada/utils/lint.py b/armada/utils/lint.py index f3ac2902..05ac5372 100644 --- a/armada/utils/lint.py +++ b/armada/utils/lint.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import requests + from armada.const import DOCUMENT_CHART, DOCUMENT_GROUP, DOCUMENT_MANIFEST from armada.const import KEYWORD_ARMADA, KEYWORD_PREFIX, KEYWORD_GROUPS, \ KEYWORD_CHARTS, KEYWORD_RELEASE @@ -93,3 +96,14 @@ def validate_chart_document(documents): KEYWORD_RELEASE, document.get('metadata').get('name'))) return True + + +def validate_manifest_url(value): + try: + return (requests.get(value).status_code == 200) + except: + return False + + +def validate_manifest_filepath(value): + return os.path.isfile(value) diff --git a/tox.ini b/tox.ini index 81c851e1..de4efeb3 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,9 @@ commands = bandit -r armada -x armada/tests -n 5 [testenv:coverage] +passenv=http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY +setenv= + VIRTUAL_ENV={envdir} commands = python -m pytest \ --cov-branch \