From b47f421abf0cf31a16219c527e850e48a658dfc4 Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Wed, 6 Dec 2017 21:44:07 +0000 Subject: [PATCH] DECKHAND-87: Deckhand API client library This PS implements the Deckhand API client library which is based off the python-novaclient code base. The client library includes managers for all the Deckhand APIs. The following features have been implemented: * Framework for API client library * Manager for each Deckhand API (buckets, revisions, etc.) * API client library documentation Tests will be added in a follow-up (once Deckhand functional tests use Keystone). Change-Id: I829a030738f42dc7ddec623d881a99ed97d04520 --- deckhand/client/__init__.py | 0 deckhand/client/base.py | 266 ++++++++++++++++++ deckhand/client/buckets.py | 38 +++ deckhand/client/client.py | 254 +++++++++++++++++ deckhand/client/exceptions.py | 128 +++++++++ deckhand/client/revisions.py | 82 ++++++ deckhand/client/tags.py | 54 ++++ deckhand/client/validations.py | 51 ++++ deckhand/control/validations.py | 11 + deckhand/db/sqlalchemy/api.py | 31 +- deckhand/engine/document_validation.py | 4 +- deckhand/errors.py | 4 +- deckhand/service.py | 2 + .../control/test_revision_tags_controller.py | 40 +++ deckhand/tests/unit/db/test_revision_tags.py | 9 - doc/source/api_client.rst | 171 +++++++++++ doc/source/api_ref.rst | 3 +- doc/source/index.rst | 1 + requirements.txt | 1 - 19 files changed, 1128 insertions(+), 22 deletions(-) create mode 100644 deckhand/client/__init__.py create mode 100644 deckhand/client/base.py create mode 100644 deckhand/client/buckets.py create mode 100644 deckhand/client/client.py create mode 100644 deckhand/client/exceptions.py create mode 100644 deckhand/client/revisions.py create mode 100644 deckhand/client/tags.py create mode 100644 deckhand/client/validations.py create mode 100644 doc/source/api_client.rst diff --git a/deckhand/client/__init__.py b/deckhand/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deckhand/client/base.py b/deckhand/client/base.py new file mode 100644 index 00000000..07337ca0 --- /dev/null +++ b/deckhand/client/base.py @@ -0,0 +1,266 @@ +# Copyright 2010 Jacob Kaplan-Moss +# Copyright 2011 OpenStack Foundation +# Copyright 2017 AT&T Intellectual Property. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Base utilities to build API operation managers and objects on top of. +""" + +import copy +import yaml + +from oslo_utils import strutils +import six +from six.moves.urllib import parse + + +def getid(obj): + """Get object's ID or object. + + Abstracts the common pattern of allowing both an object or an object's ID + as a parameter when dealing with relationships. + """ + try: + return obj.id + except AttributeError: + return obj + + +def prepare_query_string(params): + """Convert dict params to query string""" + # Transform the dict to a sequence of two-element tuples in fixed + # order, then the encoded string will be consistent in Python 2&3. + if not params: + return '' + params = sorted(params.items(), key=lambda x: x[0]) + return '?%s' % parse.urlencode(params) if params else '' + + +def get_url_with_filter(url, filters): + query_string = prepare_query_string(filters) + url = "%s%s" % (url, query_string) + return url + + +class Resource(object): + """Base class for OpenStack resources (tenant, user, etc.). + + This is pretty much just a bag for attributes. + """ + + HUMAN_ID = False + NAME_ATTR = 'name' + + def __init__(self, manager, info, loaded=False): + """Populate and bind to a manager. + + :param manager: BaseManager object + :param info: dictionary representing resource attributes + :param loaded: prevent lazy-loading if set to True + :param resp: Response or list of Response objects + """ + self.manager = manager + self._info = info or {} + self._add_details(info) + self._loaded = loaded + + def __repr__(self): + reprkeys = sorted(k + for k in self.__dict__.keys() + if k[0] != '_' and k != 'manager') + info = ", ".join("%s=%s" % (k, getattr(self, k)) for k in reprkeys) + return "<%s %s>" % (self.__class__.__name__, info) + + @property + def api_version(self): + return self.manager.api_version + + @property + def human_id(self): + """Human-readable ID which can be used for bash completion. + """ + if self.HUMAN_ID: + name = getattr(self, self.NAME_ATTR, None) + if name is not None: + return strutils.to_slug(name) + return None + + def _add_details(self, info): + for (k, v) in info.items(): + try: + setattr(self, k, v) + self._info[k] = v + except AttributeError: + # In this case we already defined the attribute on the class + pass + + def __getattr__(self, k): + if k not in self.__dict__: + # NOTE(bcwaldon): disallow lazy-loading if already loaded once + if not self.is_loaded(): + self.get() + return self.__getattr__(k) + + raise AttributeError(k) + else: + return self.__dict__[k] + + def get(self): + """Support for lazy loading details. + + Some clients, such as novaclient have the option to lazy load the + details, details which can be loaded with this function. + """ + # set_loaded() first ... so if we have to bail, we know we tried. + self.set_loaded(True) + if not hasattr(self.manager, 'get'): + return + + new = self.manager.get(self.id) + if new: + self._add_details(new._info) + + def __eq__(self, other): + if not isinstance(other, Resource): + return NotImplemented + # two resources of different types are not equal + if not isinstance(other, self.__class__): + return False + if hasattr(self, 'id') and hasattr(other, 'id'): + return self.id == other.id + return self._info == other._info + + def __ne__(self, other): + # Using not of '==' implementation because the not of + # __eq__, when it returns NotImplemented, is returning False. + return not self == other + + def is_loaded(self): + return self._loaded + + def set_loaded(self, val): + self._loaded = val + + def set_info(self, key, value): + self._info[key] = value + + def to_dict(self): + return copy.deepcopy(self._info) + + +class Manager(object): + """Manager for API service. + + Managers interact with a particular type of API (buckets, revisions, etc.) + and provide CRUD operations for them. + """ + resource_class = None + + def __init__(self, api): + self.api = api + + @property + def client(self): + return self.api.client + + @property + def api_version(self): + return self.api.api_version + + def _to_dict(self, body, many=False): + """Convert YAML-formatted response body into dict or list. + + :param body: YAML-formatted response body to convert. + :param many: Controls whether to return list or dict. If True, returns + list, else dict. False by default. + :rtype: dict or list + """ + try: + return ( + list(yaml.safe_load_all(body)) + if many else yaml.safe_load(body) + ) + except yaml.YAMLError: + return None + + def _list(self, url, response_key=None, obj_class=None, body=None, + filters=None): + if filters: + url = get_url_with_filter(url, filters) + if body: + resp, body = self.api.client.post(url, body=body) + else: + resp, body = self.api.client.get(url) + body = self._to_dict(body, many=True) + + if obj_class is None: + obj_class = self.resource_class + + if response_key is not None: + data = body[response_key] + else: + data = body + + items = [obj_class(self, res, loaded=True) + for res in data if res] + return items + + def _get(self, url, response_key=None, filters=None): + if filters: + url = get_url_with_filter(url, filters) + resp, body = self.api.client.get(url) + body = self._to_dict(body) + + if response_key is not None: + content = body[response_key] + else: + content = body + return self.resource_class(self, content, loaded=True) + + def _create(self, url, data, response_key=None): + if isinstance(data, six.string_types): + resp, body = self.api.client.post(url, body=data) + else: + resp, body = self.api.client.post(url, data=data) + body = self._to_dict(body) + + if body: + if response_key: + return self.resource_class(self, body[response_key]) + else: + return self.resource_class(self, body) + else: + return body + + def _delete(self, url): + resp, body = self.api.client.delete(url) + body = self._to_dict(body) + return body + + def _update(self, url, data, response_key=None): + if isinstance(data, six.string_types): + resp, body = self.api.client.put(url, body=data) + else: + resp, body = self.api.client.put(url, data=data) + body = self._to_dict(body) + + if body: + if response_key: + return self.resource_class(self, body[response_key]) + else: + return self.resource_class(self, body) + else: + return body diff --git a/deckhand/client/buckets.py b/deckhand/client/buckets.py new file mode 100644 index 00000000..5ce320f0 --- /dev/null +++ b/deckhand/client/buckets.py @@ -0,0 +1,38 @@ +# Copyright 2017 AT&T Intellectual Property. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from deckhand.client import base + + +class Bucket(base.Resource): + def __repr__(self): + return ("" % self.status['bucket']) + + +class BucketManager(base.Manager): + """Manage :class:`Bucket` resources.""" + resource_class = Bucket + + def update(self, bucket_name, documents): + """Create, update or delete documents associated with a bucket. + + :param str bucket_name: Gets or creates a bucket by this name. + :param str documents: YAML-formatted string of Deckhand-compatible + documents to create in the bucket. + :returns: The created documents along with their associated bucket + and revision. + """ + url = '/api/v1.0/buckets/%s/documents' % bucket_name + return self._update(url, documents) diff --git a/deckhand/client/client.py b/deckhand/client/client.py new file mode 100644 index 00000000..d8f0ef3a --- /dev/null +++ b/deckhand/client/client.py @@ -0,0 +1,254 @@ +# Copyright 2010 Jacob Kaplan-Moss +# Copyright 2011 OpenStack Foundation +# Copyright 2011 Piston Cloud Computing, Inc. +# Copyright 2017 AT&T Intellectual Property. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Deckhand Client interface. Handles the REST calls and responses. +""" + +from keystoneauth1 import adapter +from keystoneauth1 import identity +from keystoneauth1 import session as ksession +from oslo_log import log as logging + +from deckhand.client import buckets +from deckhand.client import exceptions +from deckhand.client import revisions +from deckhand.client import tags +from deckhand.client import validations + + +class SessionClient(adapter.Adapter): + """Wrapper around ``keystoneauth1`` client session implementation and used + internally by :class:`Client` below. + + Injects Deckhand-specific YAML headers necessary for communication with the + Deckhand API. + """ + + client_name = 'python-deckhandclient' + client_version = '1.0' + + def __init__(self, *args, **kwargs): + self.api_version = kwargs.pop('api_version', None) + super(SessionClient, self).__init__(*args, **kwargs) + + def request(self, url, method, **kwargs): + kwargs.setdefault('headers', kwargs.get('headers', {})) + kwargs['headers']['Accept'] = 'application/x-yaml' + kwargs['headers']['Content-Type'] = 'application/x-yaml' + + raise_exc = kwargs.pop('raise_exc', True) + kwargs['data'] = kwargs.pop('body', None) + resp = super(SessionClient, self).request(url, method, raise_exc=False, + **kwargs) + body = resp.content + + if raise_exc and resp.status_code >= 400: + raise exceptions.from_response(resp, body, url, method) + + return resp, body + + +def _construct_http_client(api_version=None, + auth=None, + auth_token=None, + auth_url=None, + cacert=None, + cert=None, + endpoint_override=None, + endpoint_type='publicURL', + http_log_debug=False, + insecure=False, + logger=None, + password=None, + project_domain_id=None, + project_domain_name=None, + project_id=None, + project_name=None, + region_name=None, + service_name=None, + service_type='deckhand', + session=None, + timeout=None, + user_agent='python-deckhandclient', + user_domain_id=None, + user_domain_name=None, + user_id=None, + username=None, + **kwargs): + if not session: + if not auth and auth_token: + auth = identity.Token(auth_url=auth_url, + token=auth_token, + project_id=project_id, + project_name=project_name, + project_domain_id=project_domain_id, + project_domain_name=project_domain_name) + elif not auth: + auth = identity.Password(username=username, + user_id=user_id, + password=password, + project_id=project_id, + project_name=project_name, + auth_url=auth_url, + project_domain_id=project_domain_id, + project_domain_name=project_domain_name, + user_domain_id=user_domain_id, + user_domain_name=user_domain_name) + session = ksession.Session(auth=auth, + verify=(cacert or not insecure), + timeout=timeout, + cert=cert, + user_agent=user_agent) + + return SessionClient(api_version=api_version, + auth=auth, + endpoint_override=endpoint_override, + interface=endpoint_type, + logger=logger, + region_name=region_name, + service_name=service_name, + service_type=service_type, + session=session, + user_agent=user_agent, + **kwargs) + + +class Client(object): + """Top-level object to access the Deckhand API.""" + + def __init__(self, + api_version=None, + auth=None, + auth_token=None, + auth_url=None, + cacert=None, + cert=None, + direct_use=True, + endpoint_override=None, + endpoint_type='publicURL', + http_log_debug=False, + insecure=False, + logger=None, + password=None, + project_domain_id=None, + project_domain_name=None, + project_id=None, + project_name=None, + region_name=None, + service_name=None, + service_type='deckhand', + session=None, + timeout=None, + user_domain_id=None, + user_domain_name=None, + user_id=None, + username=None, + **kwargs): + """Initialization of Client object. + + :param api_version: Compute API version + :type api_version: novaclient.api_versions.APIVersion + :param str auth: Auth + :param str auth_token: Auth token + :param str auth_url: Auth URL + :param str cacert: ca-certificate + :param str cert: certificate + :param bool direct_use: Inner variable of novaclient. Do not use it + outside novaclient. It's restricted. + :param str endpoint_override: Bypass URL + :param str endpoint_type: Endpoint Type + :param bool http_log_debug: Enable debugging for HTTP connections + :param bool insecure: Allow insecure + :param logging.Logger logger: Logger instance to be used for all + logging stuff + :param str password: User password + :param str project_domain_id: ID of project domain + :param str project_domain_name: Name of project domain + :param str project_id: Project/Tenant ID + :param str project_name: Project/Tenant name + :param str region_name: Region Name + :param str service_name: Service Name + :param str service_type: Service Type + :param str session: Session + :param float timeout: API timeout, None or 0 disables + :param str user_domain_id: ID of user domain + :param str user_domain_name: Name of user domain + :param str user_id: User ID + :param str username: Username + """ + + self.project_id = project_id + self.project_name = project_name + self.user_id = user_id + + self.logger = logger or logging.getLogger(__name__) + + self.buckets = buckets.BucketManager(self) + self.revisions = revisions.RevisionManager(self) + self.tags = tags.RevisionTagManager(self) + self.validations = validations.ValidationManager(self) + + self.client = _construct_http_client( + api_version=api_version, + auth=auth, + auth_token=auth_token, + auth_url=auth_url, + cacert=cacert, + cert=cert, + endpoint_override=endpoint_override, + endpoint_type=endpoint_type, + http_log_debug=http_log_debug, + insecure=insecure, + logger=self.logger, + password=password, + project_domain_id=project_domain_id, + project_domain_name=project_domain_name, + project_id=project_id, + project_name=project_name, + region_name=region_name, + service_name=service_name, + service_type=service_type, + session=session, + timeout=timeout, + user_domain_id=user_domain_id, + user_domain_name=user_domain_name, + user_id=user_id, + username=username, + **kwargs) + + @property + def api_version(self): + return self.client.api_version + + @api_version.setter + def api_version(self, value): + self.client.api_version = value + + @property + def projectid(self): + self.logger.warning(_("Property 'projectid' is deprecated since " + "Ocata. Use 'project_name' instead.")) + return self.project_name + + @property + def tenant_id(self): + self.logger.warning(_("Property 'tenant_id' is deprecated since " + "Ocata. Use 'project_id' instead.")) + return self.project_id diff --git a/deckhand/client/exceptions.py b/deckhand/client/exceptions.py new file mode 100644 index 00000000..db7c7af1 --- /dev/null +++ b/deckhand/client/exceptions.py @@ -0,0 +1,128 @@ +# Copyright 2010 Jacob Kaplan-Moss +# Copyright 2017 AT&T Intellectual Property. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Exception definitions. +""" + +import logging +import yaml + +import six + +LOG = logging.getLogger(__name__) + + +class ClientException(Exception): + """The base exception class for all exceptions this library raises.""" + message = "Unknown Error" + + def __init__(self, code, url, method, message=None, details=None, + reason=None, apiVersion=None, retry=False, status=None, + kind=None, metadata=None): + self.code = code + self.url = url + self.method = method + self.message = message or self.__class__.message + self.details = details + self.reason = reason + self.apiVersion = apiVersion + self.retry = retry + self.status = status + self.kind = kind + self.metadata = metadata + + def __str__(self): + formatted_string = "%s (HTTP %s)" % (self.message, self.code) + return formatted_string + + +class BadRequest(ClientException): + """HTTP 400 - Bad request: you sent some malformed data.""" + http_status = 400 + message = "Bad request" + + +class Unauthorized(ClientException): + """HTTP 401 - Unauthorized: bad credentials.""" + http_status = 401 + message = "Unauthorized" + + +class Forbidden(ClientException): + """HTTP 403 - Forbidden: your credentials don't give you access to this + resource. + """ + http_status = 403 + message = "Forbidden" + + +class NotFound(ClientException): + """HTTP 404 - Not found""" + http_status = 404 + message = "Not found" + + +class MethodNotAllowed(ClientException): + """HTTP 405 - Method Not Allowed""" + http_status = 405 + message = "Method Not Allowed" + + +class Conflict(ClientException): + """HTTP 409 - Conflict""" + http_status = 409 + message = "Conflict" + + +# NotImplemented is a python keyword. +class HTTPNotImplemented(ClientException): + """HTTP 501 - Not Implemented: the server does not support this operation. + """ + http_status = 501 + message = "Not Implemented" + + +_code_map = dict((c.http_status, c) + for c in ClientException.__subclasses__()) + + +def from_response(response, body, url, method=None): + """Return an instance of a ``ClientException`` or subclass based on a + request's response. + + Usage:: + + resp, body = requests.request(...) + if resp.status_code != 200: + raise exception.from_response(resp, rest.text) + """ + cls = _code_map.get(response.status_code, ClientException) + + try: + body = yaml.safe_load(body) + except yaml.YAMLError as e: + body = {} + LOG.debug('Could not convert error from server into dict: %s', + six.text_type(e)) + + kwargs = body + kwargs.update({ + 'code': response.status_code, + 'method': method, + 'url': url, + }) + + return cls(**kwargs) diff --git a/deckhand/client/revisions.py b/deckhand/client/revisions.py new file mode 100644 index 00000000..195221dd --- /dev/null +++ b/deckhand/client/revisions.py @@ -0,0 +1,82 @@ +# Copyright 2017 AT&T Intellectual Property. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from deckhand.client import base + + +class Revision(base.Resource): + def __repr__(self): + if hasattr(self, 'results'): + return ', '.join( + ["" % r['id'] for r in self.results]) + else: + try: + return ("" % base.getid(self)) + except Exception: + return ("") + + +class RevisionManager(base.Manager): + """Manage :class:`Revision` resources.""" + resource_class = Revision + + def list(self, **filters): + """Get a list of revisions.""" + url = '/api/v1.0/revisions' + # Call `_get` instead of `_list` because the response from the server + # is a dict of form `{"count": n, "results": []}`. + return self._get(url, filters=filters) + + def get(self, revision_id): + """Get details for a revision.""" + url = '/api/v1.0/revisions/%s' % revision_id + return self._get(url) + + def diff(self, revision_id, comparison_revision_id): + """Get revision diff between two revisions.""" + url = '/api/v1.0/revisions/%s/diff/%s' % ( + revision_id, comparison_revision_id) + return self._get(url) + + def rollback(self, revision_id): + """Rollback to a previous revision, effectively creating a new one.""" + url = '/api/v1.0/rollback/%s' % revision_id + return self._post(url) + + def documents(self, revision_id, rendered=True, **filters): + """Get a list of revision documents or rendered documents. + + :param int revision_id: Revision ID. + :param bool rendered: If True, returns list of rendered documents. + Else returns list of unmodified, raw documents. + :param filters: Filters to apply to response body. + :returns: List of documents or rendered documents. + :rtype: list[:class:`Revision`] + """ + if rendered: + url = '/api/v1.0/revisions/%s/rendered-documents' % revision_id + else: + url = '/api/v1.0/revisions/%s/documents' % revision_id + return self._list(url, filters=filters) + + def delete_all(self): + """Delete all revisions. + + .. warning:: + + Effectively the same as purging the entire database. + """ + url = '/api/v1.0/revisions' + return self._delete(url) diff --git a/deckhand/client/tags.py b/deckhand/client/tags.py new file mode 100644 index 00000000..f8cf0939 --- /dev/null +++ b/deckhand/client/tags.py @@ -0,0 +1,54 @@ +# Copyright 2017 AT&T Intellectual Property. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from deckhand.client import base + + +class RevisionTag(base.Resource): + def __repr__(self): + try: + return ("" % self.tag) + except AttributeError: + return ("") + + +class RevisionTagManager(base.Manager): + """Manage :class:`RevisionTag` resources.""" + resource_class = RevisionTag + + def list(self, revision_id): + """Get list of revision tags.""" + url = '/api/v1.0/revisions/%s/tags' % revision_id + return self._list(url) + + def get(self, revision_id, tag): + """Get details for a revision tag.""" + url = '/api/v1.0/revisions/%s/tags/%s' % (revision_id, tag) + return self._get(url) + + def create(self, revision_id, tag, data=None): + """Create a revision tag.""" + url = '/api/v1.0/revisions/%s/tags/%s' % (revision_id, tag) + return self._create(url, data=data) + + def delete(self, revision_id, tag): + """Delete a revision tag.""" + url = '/api/v1.0/revisions/%s/tags/%s' % (revision_id, tag) + return self._delete(url) + + def delete_all(self, revision_id): + """Delete all revision tags.""" + url = '/api/v1.0/revisions/%s/tags' % revision_id + return self._delete(url) diff --git a/deckhand/client/validations.py b/deckhand/client/validations.py new file mode 100644 index 00000000..8720a239 --- /dev/null +++ b/deckhand/client/validations.py @@ -0,0 +1,51 @@ +# Copyright 2017 AT&T Intellectual Property. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from deckhand.client import base + + +class Validation(base.Resource): + def __repr__(self): + return ("") + + +class ValidationManager(base.Manager): + """Manage :class:`Validation` resources.""" + resource_class = Validation + + def list(self, revision_id): + """Get list of revision validations.""" + url = '/api/v1.0/revisions/%s/validations' % revision_id + return self._list(url) + + def list_entries(self, revision_id, validation_name): + """Get list of entries for a validation.""" + url = '/api/v1.0/revisions/%s/validations/%s' % (revision_id, + validation_name) + # Call `_get` instead of `_list` because the response from the server + # is a dict of form `{"count": n, "results": []}`. + return self._get(url) + + def get_entry(self, revision_id, validation_name, entry_id): + """Get entry details for a validation.""" + url = '/api/v1.0/revisions/%s/validations/%s/entries/%s' % ( + revision_id, validation_name, entry_id) + return self._get(url) + + def create(self, revision_id, validation_name, data): + """Associate a validation with a revision.""" + url = '/api/v1.0/revisions/%s/validations/%s' % (revision_id, + validation_name) + return self._create(url, data=data) diff --git a/deckhand/control/validations.py b/deckhand/control/validations.py index 26689c36..74bfe70b 100644 --- a/deckhand/control/validations.py +++ b/deckhand/control/validations.py @@ -43,6 +43,17 @@ class ValidationsResource(api_base.BaseResource): LOG.error(error_msg) raise falcon.HTTPBadRequest(description=six.text_type(e)) + if not validation_data: + error_msg = 'Validation payload must be provided.' + LOG.error(error_msg) + raise falcon.HTTPBadRequest(description=error_msg) + + if not all([validation_data.get(x) for x in ('status', 'validator')]): + error_msg = 'Validation payload must contain keys: %s.' % ( + ', '.join(['"status"', '"validator"'])) + LOG.error(error_msg) + raise falcon.HTTPBadRequest(description=error_msg) + try: resp_body = db_api.validation_create( revision_id, validation_name, validation_data) diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index d5f07a38..5e111199 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -94,6 +94,17 @@ def setup_db(): def raw_query(query, **kwargs): """Execute a raw query against the database.""" + + # Cast all the strings that represent integers to integers because type + # matters when using ``bindparams``. + for key, val in kwargs.items(): + if key.endswith('_id'): + try: + val = int(val) + kwargs[key] = val + except ValueError: + pass + stmt = text(query) stmt = stmt.bindparams(**kwargs) return get_engine().execute(stmt) @@ -926,7 +937,13 @@ def revision_tag_create(revision_id, tag, data=None, session=None): tag_model.save(session=session) resp = tag_model.to_dict() except db_exception.DBDuplicateEntry: - resp = None + # Update the revision tag if it already exists. + tag_to_update = session.query(models.RevisionTag)\ + .filter_by(tag=tag, revision_id=revision_id)\ + .one() + tag_to_update.update({'data': data}) + tag_to_update.save(session=session) + resp = tag_to_update.to_dict() return resp @@ -980,11 +997,10 @@ def revision_tag_delete(revision_id, tag, session=None): :param session: Database session object. :returns: None """ - session = session or get_session() - result = session.query(models.RevisionTag)\ - .filter_by(tag=tag, revision_id=revision_id)\ - .delete(synchronize_session=False) - if result == 0: + query = raw_query( + """DELETE FROM revision_tags WHERE tag=:tag AND + revision_id=:revision_id;""", tag=tag, revision_id=revision_id) + if query.rowcount == 0: raise errors.RevisionTagNotFound(tag=tag, revision=revision_id) @@ -1111,9 +1127,10 @@ def validation_get_all(revision_id, session=None): # has its own validation but for this query we want to return the result # of the overall validation for the revision. If just 1 document failed # validation, we regard the validation for the whole revision as 'failure'. + query = raw_query(""" SELECT DISTINCT name, status FROM validations as v1 - WHERE revision_id = :revision_id AND status = ( + WHERE revision_id=:revision_id AND status = ( SELECT status FROM validations as v2 WHERE v2.name = v1.name ORDER BY status diff --git a/deckhand/engine/document_validation.py b/deckhand/engine/document_validation.py index 654d0e24..495c0411 100644 --- a/deckhand/engine/document_validation.py +++ b/deckhand/engine/document_validation.py @@ -163,7 +163,7 @@ class DocumentValidation(object): :results: The validation results generated during document validation. :type results: list[dict] :returns: List of formatted validation results. - :rtype: list[dict] + :rtype: `func`:list[dict] """ internal_validator = { 'name': 'deckhand', @@ -275,7 +275,7 @@ class DocumentValidation(object): later. :returns: A list of validations (one for each document validated). - :rtype: list[dict] + :rtype: `func`:list[dict] :raises errors.InvalidDocumentFormat: If the document failed schema validation and the failure is deemed critical. :raises errors.InvalidDocumentSchema: If no JSON schema for could be diff --git a/deckhand/errors.py b/deckhand/errors.py index 30acd8bc..a089157e 100644 --- a/deckhand/errors.py +++ b/deckhand/errors.py @@ -245,8 +245,8 @@ class RevisionNotFound(DeckhandException): class RevisionTagNotFound(DeckhandException): - msg_fmt = ("The requested tag %(tag)s for revision %(revision)s was not " - "found.") + msg_fmt = ("The requested tag '%(tag)s' for revision %(revision)s was " + "not found.") code = 404 diff --git a/deckhand/service.py b/deckhand/service.py index e901db7a..1de32bce 100644 --- a/deckhand/service.py +++ b/deckhand/service.py @@ -1,3 +1,5 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at diff --git a/deckhand/tests/unit/control/test_revision_tags_controller.py b/deckhand/tests/unit/control/test_revision_tags_controller.py index e3d5708f..49af3192 100644 --- a/deckhand/tests/unit/control/test_revision_tags_controller.py +++ b/deckhand/tests/unit/control/test_revision_tags_controller.py @@ -18,6 +18,46 @@ from deckhand import factories from deckhand.tests.unit.control import base as test_base +class TestRevisionTagsController(test_base.BaseControllerTest): + + def setUp(self): + super(TestRevisionTagsController, self).setUp() + rules = {'deckhand:create_cleartext_documents': '@'} + self.policy.set_rules(rules) + + # Create a revision to tag. + secrets_factory = factories.DocumentSecretFactory() + payload = [secrets_factory.gen_test('Certificate', 'cleartext')] + resp = self.app.simulate_put( + '/api/v1.0/buckets/mop/documents', + headers={'Content-Type': 'application/x-yaml'}, + body=yaml.safe_dump_all(payload)) + self.assertEqual(200, resp.status_code) + self.revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][ + 'revision'] + + def test_delete_tag(self): + rules = {'deckhand:create_tag': '@', + 'deckhand:delete_tag': '@', + 'deckhand:show_tag': '@'} + self.policy.set_rules(rules) + + resp = self.app.simulate_post( + '/api/v1.0/revisions/%s/tags/%s' % (self.revision_id, 'test'), + headers={'Content-Type': 'application/x-yaml'}) + self.assertEqual(201, resp.status_code) + + resp = self.app.simulate_delete( + '/api/v1.0/revisions/%s/tags/%s' % (self.revision_id, 'test'), + headers={'Content-Type': 'application/x-yaml'}) + self.assertEqual(204, resp.status_code) + + resp = self.app.simulate_get( + '/api/v1.0/revisions/%s/tags/%s' % (self.revision_id, 'test'), + headers={'Content-Type': 'application/x-yaml'}) + self.assertEqual(404, resp.status_code) + + class TestRevisionTagsControllerNegativeRBAC(test_base.BaseControllerTest): """Test suite for validating negative RBAC scenarios for revision tags controller. diff --git a/deckhand/tests/unit/db/test_revision_tags.py b/deckhand/tests/unit/db/test_revision_tags.py index 459c043e..b9b00476 100644 --- a/deckhand/tests/unit/db/test_revision_tags.py +++ b/deckhand/tests/unit/db/test_revision_tags.py @@ -106,12 +106,3 @@ class TestRevisionTags(base.TestDbBase): # Validate that deleting all tags without any tags doesn't raise # errors. db_api.revision_tag_delete_all(self.revision_id) - - def test_create_duplicate_tag(self): - tag = test_utils.rand_name(self.__class__.__name__ + '-Tag') - # Create the same tag twice and validate that it returns None the - # second time. - - db_api.revision_tag_create(self.revision_id, tag) - resp = db_api.revision_tag_create(self.revision_id, tag) - self.assertIsNone(resp) diff --git a/doc/source/api_client.rst b/doc/source/api_client.rst new file mode 100644 index 00000000..2f75311f --- /dev/null +++ b/doc/source/api_client.rst @@ -0,0 +1,171 @@ +.. + Copyright 2017 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _api-client-library: + +Deckhand API Client Library Documentation +========================================= + +The recommended approach to instantiate the Deckhand client is via a Keystone +session: + +:: + + from keystoneauth1.identity import v3 + from keystoneauth1 import session + + keystone_auth = { + 'project_domain_name': PROJECT_DOMAIN_NAME, + 'project_name': PROJECT_NAME, + 'user_domain_name': USER_DOMAIN_NAME, + 'password': PASSWORD, + 'username': USERNAME, + 'auth_url': AUTH_URL, + } + auth = v3.Password(**keystone_auth) + sess = session.Session(auth=auth) + deckhandclient = client.Client(session=sess) + +Alternatively, one can use non-session authentication to instantiate the +client, though this approach has been `deprecated`_. + +:: + + from deckhand.client import client + + deckhandclient = client.Client( + username=USERNAME, + password=PASSWORD, + project_name=PROECT_NAME, + project_domain_name=PROJECT_DOMAIN_NAME, + user_domain_name=USER_DOMAIN_NAME, + auth_url=AUTH_URL) + +.. _deprecated: https://docs.openstack.org/python-keystoneclient/latest/using-api-v3.html#non-session-authentication-deprecated + +.. note:: + + The Deckhand client by default expects that the service be registered + under the Keystone service catalog as ``deckhand``. To provide a different + value pass ``service_type=SERVICE_TYPE`` to the ``Client`` constructor. + +After you have instantiated an instance of the Deckhand client, you can invoke +the client managers' functionality: + +:: + + # Generate a sample document. + payload = """ + --- + schema: deckhand/Certificate/v1.0 + metadata: + schema: metadata/Document/v1.0 + name: application-api + storagePolicy: cleartext + data: |- + -----BEGIN CERTIFICATE----- + MIIDYDCCAkigAwIBAgIUKG41PW4VtiphzASAMY4/3hL8OtAwDQYJKoZIhvcNAQEL + ...snip... + P3WT9CfFARnsw2nKjnglQcwKkKLYip0WY2wh3FE7nrQZP6xKNaSRlh6p2pCGwwwH + HkvVwA== + -----END CERTIFICATE----- + """ + + # Create a bucket and associate it with the document. + result = client.buckets.update('mop', payload) + + >>> result + + + # Convert the response to a dictionary. + >>> result.to_dict() + {'status': {'bucket': 'mop', 'revision': 1}, + 'schema': 'deckhand/Certificate/v1.0', 'data': {...} 'id': 1, + 'metadata': {'layeringDefinition': {'abstract': False}, + 'storagePolicy': 'cleartext', 'name': 'application-api', + 'schema': 'metadata/Document/v1.0'}} + + # Show the revision that was created. + revision = client.revisions.get(1) + + >>> revision.to_dict() + {'status': 'success', 'tags': {}, + 'url': 'https://deckhand/api/v1.0/revisions/1', + 'buckets': ['mop'], 'validationPolicies': [], 'id': 1, + 'createdAt': '2017-12-09T00:15:04.309071'} + + # List all revisions. + revisions = client.revisions.list() + + >>> revisions.to_dict() + {'count': 1, 'results': [{'buckets': ['mop'], 'id': 1, + 'createdAt': '2017-12-09T00:29:34.031460', 'tags': []}]} + + # List raw documents for the created revision. + raw_documents = client.revisions.documents(1, rendered=False) + + >>> [r.to_dict() for r in raw_documents] + [{'status': {'bucket': 'foo', 'revision': 1}, + 'schema': 'deckhand/Certificate/v1.0', 'data': {...}, 'id': 1, + 'metadata': {'layeringDefinition': {'abstract': False}, + 'storagePolicy': 'cleartext', 'name': 'application-api', + 'schema': 'metadata/Document/v1.0'}}] + +Client Reference +---------------- + +For more information about how to use the Deckhand client, refer to the +following documentation: + +.. autoclass:: deckhand.client.client.SessionClient + :members: + +.. autoclass:: deckhand.client.client.Client + :members: + +Manager Reference +----------------- + +For more information about how to use the client managers, refer to the +following documentation: + +.. autoclass:: deckhand.client.buckets.Bucket + :members: + +.. autoclass:: deckhand.client.buckets.BucketManager + :members: + :undoc-members: + +.. autoclass:: deckhand.client.revisions.Revision + :members: + +.. autoclass:: deckhand.client.revisions.RevisionManager + :members: + :undoc-members: + +.. autoclass:: deckhand.client.tags.RevisionTag + :members: + +.. autoclass:: deckhand.client.tags.RevisionTagManager + :members: + :undoc-members: + +.. autoclass:: deckhand.client.validations.Validation + :members: + +.. autoclass:: deckhand.client.validations.ValidationManager + :members: + :undoc-members: diff --git a/doc/source/api_ref.rst b/doc/source/api_ref.rst index 1273debc..10331172 100644 --- a/doc/source/api_ref.rst +++ b/doc/source/api_ref.rst @@ -381,7 +381,8 @@ POST ``/revisions/{{revision_id}}/tags/{{tag}}`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Associate the revision with a collection of metadata, if provided, by way of -a tag. The tag itself can be used to label the revision. +a tag. The tag itself can be used to label the revision. If a tag by name +``tag`` already exists, the tag's associated metadata is updated. Sample request with body: diff --git a/doc/source/index.rst b/doc/source/index.rst index 0f14bfef..fe69fbc3 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -46,6 +46,7 @@ User's Guide layering revision_history api_ref + api_client Developer's Guide ================= diff --git a/requirements.txt b/requirements.txt index ce992d9f..6ae2c036 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ Routes>=2.3.1 # MIT keystoneauth1>=3.2.0 # Apache-2.0 six>=1.9.0 # MIT -oslo.concurrency>=3.8.0 # Apache-2.0 stevedore>=1.20.0 # Apache-2.0 python-keystoneclient>=3.8.0 # Apache-2.0 python-memcached==1.58