diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index 16f94acc..e2c6c83b 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -128,9 +128,6 @@ def require_unique_document_schema(schema=None): existing_documents = revision_get_documents( schema=schema, deleted=False, include_history=False) existing_document_names = [x['name'] for x in existing_documents] - # `conflict_names` is calculated by checking whether any documents - # in `documents` is a layering policy with a name not found in - # `existing_documents`. conflicting_names = [ x['metadata']['name'] for x in documents if x['metadata']['name'] not in existing_document_names and @@ -250,7 +247,7 @@ def _documents_create(bucket_name, values_list, session=None): try: existing_document = document_get( - raw_dict=True, deleted=False, + raw_dict=True, deleted=False, revision_id='latest', **{x: values[x] for x in filters}) except errors.DocumentNotFound: # Ignore bad data at this point. Allow creation to bubble up the @@ -311,12 +308,14 @@ def _make_hash(data): json.dumps(data, sort_keys=True).encode('utf-8')).hexdigest() -def document_get(session=None, raw_dict=False, **filters): - """Retrieve a document from the DB. +def document_get(session=None, raw_dict=False, revision_id=None, **filters): + """Retrieve the first document for ``revision_id`` that match ``filters``. :param session: Database session object. :param raw_dict: Whether to retrieve the exact way the data is stored in DB if ``True``, else the way users expect the data. + :param revision_id: The ID corresponding to the ``Revision`` object. If the + it is "latest", then retrieve the latest revision, if one exists. :param filters: Dictionary attributes (including nested) used to filter out revision documents. :returns: Dictionary representation of retrieved document. @@ -324,6 +323,15 @@ def document_get(session=None, raw_dict=False, **filters): """ session = session or get_session() + if revision_id == 'latest': + revision = session.query(models.Revision)\ + .order_by(models.Revision.created_at.desc())\ + .first() + if revision: + filters['revision_id'] = revision.id + elif revision_id: + filters['revision_id'] = revision_id + # TODO(fmontei): Currently Deckhand doesn't support filtering by nested # JSON fields via sqlalchemy. For now, filter the documents using all # "regular" filters via sqlalchemy and all nested filters via Python. @@ -357,21 +365,20 @@ def document_get_all(session=None, raw_dict=False, revision_id=None, :param raw_dict: Whether to retrieve the exact way the data is stored in DB if ``True``, else the way users expect the data. :param revision_id: The ID corresponding to the ``Revision`` object. If the - ID is ``None``, then retrieve the latest revision, if one exists. + it is "latest", then retrieve the latest revision, if one exists. :param filters: Dictionary attributes (including nested) used to filter out revision documents. :returns: Dictionary representation of each retrieved document. """ session = session or get_session() - if revision_id is None: - # If no revision_id is specified, grab the newest one. + if revision_id == 'latest': revision = session.query(models.Revision)\ .order_by(models.Revision.created_at.desc())\ .first() if revision: filters['revision_id'] = revision.id - else: + elif revision_id: filters['revision_id'] = revision_id # TODO(fmontei): Currently Deckhand doesn't support filtering by nested @@ -452,7 +459,7 @@ def revision_get(revision_id=None, session=None): :param revision_id: The ID corresponding to the ``Revision`` object. :param session: Database session object. :returns: Dictionary representation of retrieved revision. - :raises: RevisionNotFound if the revision was not found. + :raises RevisionNotFound: if the revision was not found. """ session = session or get_session() @@ -474,7 +481,7 @@ def revision_get_latest(session=None): :param session: Database session object. :returns: Dictionary representation of latest revision. - :raises: RevisionNotFound if the latest revision was not found. + :raises RevisionNotFound: if the latest revision was not found. """ session = session or get_session() @@ -647,7 +654,7 @@ def _filter_revision_documents(documents, unique_only, **filters): for document in documents: # NOTE(fmontei): Only want to include non-validation policy documents # for this endpoint. - if document['schema'] == types.VALIDATION_POLICY_SCHEMA: + if document['schema'].startswith(types.VALIDATION_POLICY_SCHEMA): continue if _apply_filters(document, **filters): @@ -682,7 +689,7 @@ def revision_get_documents(revision_id=None, include_history=True, :param filters: Key-value pairs used for filtering out revision documents. :returns: All revision documents for ``revision_id`` that match the ``filters``, including document revision history if applicable. - :raises: RevisionNotFound if the revision was not found. + :raises RevisionNotFound: if the revision was not found. """ session = session or get_session() revision_documents = [] diff --git a/deckhand/engine/document_validation.py b/deckhand/engine/document_validation.py index 8dddcea3..5194ec2f 100644 --- a/deckhand/engine/document_validation.py +++ b/deckhand/engine/document_validation.py @@ -85,7 +85,7 @@ class DocumentValidation(object): been registered by external services via ``DataSchema`` documents. """ data_schemas = db_api.document_get_all( - schema=types.DATA_SCHEMA_SCHEMA) + schema=types.DATA_SCHEMA_SCHEMA, revision_id='latest') for data_schema in data_schemas: if cls.schema_re.match(data_schema['metadata']['name']): diff --git a/deckhand/engine/secrets_manager.py b/deckhand/engine/secrets_manager.py index 0761224f..8519c820 100644 --- a/deckhand/engine/secrets_manager.py +++ b/deckhand/engine/secrets_manager.py @@ -142,7 +142,7 @@ class SecretsSubstitution(object): if src_path == '.': src_path = '.secret' - # TODO(fmontei): Use secrets_manager for this logic. Need to + # TODO(fmontei): Use SecretsManager for this logic. Need to # check Barbican for the secret if it has been encrypted. src_doc = db_api.document_get( schema=src_schema, name=src_name, is_secret=True, diff --git a/deckhand/tests/unit/control/test_buckets_controller.py b/deckhand/tests/unit/control/test_buckets_controller.py index 60ae9ca1..b588f5b5 100644 --- a/deckhand/tests/unit/control/test_buckets_controller.py +++ b/deckhand/tests/unit/control/test_buckets_controller.py @@ -110,6 +110,47 @@ class TestBucketsController(test_base.BaseControllerTest): 'secret': payload[-1]['data']} _do_test([payload[-1]]) + def test_create_delete_then_recreate_document_in_different_bucket(self): + """Ordiniarly creating a document with the same metadata.name/schema + in a separate bucket raises an exception, but if we delete the document + and re-create it in a different bucket this should be a success + scenario. + """ + rules = {'deckhand:create_cleartext_documents': '@'} + self.policy.set_rules(rules) + + payload = factories.DocumentFactory(2, [1, 1]).gen_test({}) + bucket_name = test_utils.rand_name('bucket') + alt_bucket_name = test_utils.rand_name('bucket') + + # Create the documents in the first bucket. + resp = self.app.simulate_put( + '/api/v1.0/buckets/%s/documents' % bucket_name, + headers={'Content-Type': 'application/x-yaml'}, + body=yaml.safe_dump_all(payload)) + self.assertEqual(200, resp.status_code) + documents = list(yaml.safe_load_all(resp.text)) + self.assertEqual(3, len(documents)) + self.assertEqual([bucket_name] * 3, + [d['status']['bucket'] for d in documents]) + + # Delete the documents from the first bucket. + resp = self.app.simulate_put( + '/api/v1.0/buckets/%s/documents' % bucket_name, + headers={'Content-Type': 'application/x-yaml'}, body=None) + self.assertEqual(200, resp.status_code) + + # Re-create the documents in the second bucket. + resp = self.app.simulate_put( + '/api/v1.0/buckets/%s/documents' % alt_bucket_name, + headers={'Content-Type': 'application/x-yaml'}, + body=yaml.safe_dump_all(payload)) + self.assertEqual(200, resp.status_code) + documents = list(yaml.safe_load_all(resp.text)) + self.assertEqual(3, len(documents)) + self.assertEqual([alt_bucket_name] * 3, + [d['status']['bucket'] for d in documents]) + class TestBucketsControllerNegative(test_base.BaseControllerTest): """Test suite for validating negative scenarios for bucket controller.""" @@ -162,6 +203,25 @@ schema: resp_error = ' '.join(resp.text.split()) self.assertRegexpMatches(resp_error, error_re) + def test_put_conflicting_document(self): + rules = {'deckhand:create_cleartext_documents': '@'} + self.policy.set_rules(rules) + + payload = factories.DocumentFactory(1, [1]).gen_test({})[0] + bucket_name = test_utils.rand_name('bucket') + alt_bucket_name = test_utils.rand_name('bucket') + # Create document in `bucket_name`. + resp = self.app.simulate_put( + '/api/v1.0/buckets/%s/documents' % bucket_name, + headers={'Content-Type': 'application/x-yaml'}, + body=yaml.safe_dump_all([payload])) + # Create same document in `alt_bucket_name` and validate conflict. + resp = self.app.simulate_put( + '/api/v1.0/buckets/%s/documents' % alt_bucket_name, + headers={'Content-Type': 'application/x-yaml'}, + body=yaml.safe_dump_all([payload])) + self.assertEqual(409, resp.status_code) + class TestBucketsControllerNegativeRBAC(test_base.BaseControllerTest): """Test suite for validating negative RBAC scenarios for bucket diff --git a/doc/source/validation.rst b/doc/source/validation.rst index 5ca38f83..04fb694a 100644 --- a/doc/source/validation.rst +++ b/doc/source/validation.rst @@ -26,19 +26,14 @@ The validation system provides a unified approach to complex validations that require coordination of multiple documents and business logic that resides in consumer services. -Services can report success or failure of named validations for a given -revision. Those validations can then be referenced by many ``ValidationPolicy`` -control documents. The intended purpose use is to allow a simple mapping that -enables consuming services to be able to quickly check whether the -configuration in Deckhand is in a valid state for performing a specific -action. +Deckhand focuses on two types of validations: schema validations and policy +validations. Deckhand-Provided Validations ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In addition to allowing 3rd party services to report configurable validation -statuses, Deckhand provides a few internal validations which are made -available immediately upon document ingestion. +Deckhand provides a few internal validations which are made available +immediately upon document ingestion. Here is a list of internal validations: @@ -59,3 +54,61 @@ documents are used to register a new validation mapping that other services can reference to verify whether a Deckhand bucket is in a valid configuration. For more information, refer to the ``DataSchema`` section in :ref:`document-types`. + +Schema Validations +^^^^^^^^^^^^^^^^^^ + +Schema validations are controlled by two mechanisms: + +1) Deckhand's internal schema validation for sanity-checking the formatting + of the default documents that it understands. For example, Deckhand + will check that a ``LayeringPolicy``, ``ValidationPolicy`` or ``DataSchema`` + adhere to the "skeleton" or schemas registered under + ``deckhand.engine.schema``. + + .. note:: + + Each document is always subjected to 2 stages of document validation: + the first stage checks whether the document adheres to the fundamental + building blocks: Does it have a ``schema``, ``metadata``, and ``data`` + section? The second stage then checks whether the document's ``schema`` + passes a more nuanced schema validation specific to that ``schema``. + +2) Externally provided validations via ``DataSchema`` documents. These + documents can be registered by external services and subject the target + document's data section to *additional* validations, validations specified + by the ``data`` section of the ``DataSchema`` document. + + For more information about ``DataSchema`` documents, please refer to + :ref:`document-types`. + +Policy Validations +^^^^^^^^^^^^^^^^^^ + +*Not yet implemented*. + +Validation Policies +^^^^^^^^^^^^^^^^^^^ + +Validation policies allow services to report success or failure of named +validations for a given revision. Those validations can then be referenced by +many ``ValidationPolicy`` control documents. The intended purpose use is to +allow a simple mapping that enables consuming services to be able to quickly +check whether the configuration in Deckhand is in a valid state for performing +a specific action. + +.. note:: + + ``ValidationPolicy`` documents are not the same as ``DataSchema`` documents. + A ``ValidationPolicy`` document can reference a list of internal Deckhand + validations in addition to externally registered ``DataSchema`` documents. + Once all the validations specified in the ``ValidationPolicy`` are executed + and succeed, then services can check whether the documents in a bucket are + stable, in accordance with the ``ValidationPolicy``. + +Validation Module +----------------- + +.. autoclass:: deckhand.engine.document_validation.DocumentValidation + :members: + :private-members: