Merge "Validate bucket diffing works with revision rollback"

This commit is contained in:
Zuul 2018-10-29 17:26:38 +00:00 committed by Gerrit Code Review
commit 464d2c0ea5
17 changed files with 179 additions and 173 deletions

View File

@ -102,8 +102,8 @@ def revision_diff(revision_id, comparison_revision_id, deepdiff=False):
bucket_a: created
"""
if deepdiff:
docs = (_rendered_doc(revision_id) if revision_id != 0 else [])
comparison_docs = (_rendered_doc(comparison_revision_id)
docs = (_render_documents(revision_id) if revision_id != 0 else [])
comparison_docs = (_render_documents(comparison_revision_id)
if comparison_revision_id != 0 else [])
else:
# Retrieve document history for each revision. Since `revision_id` of 0
@ -143,7 +143,7 @@ def revision_diff(revision_id, comparison_revision_id, deepdiff=False):
shared_buckets = set(buckets.keys()).intersection(
comparison_buckets.keys())
# `unshared_buckets` references buckets not shared by both `revision_id`
# and `comparison_revision_id` -- i.e. their non-intersection.
# and `comparison_revision_id` -- i.e. their union.
unshared_buckets = set(buckets.keys()).union(
comparison_buckets.keys()) - shared_buckets
@ -163,9 +163,8 @@ def revision_diff(revision_id, comparison_revision_id, deepdiff=False):
result[bucket_name] = 'unmodified'
else:
result[bucket_name] = 'modified'
# If deepdiff enabled
# If deepdiff is enabled, find out diff between buckets
if deepdiff:
# find out diff between buckets
bucket_diff = _diff_buckets(buckets[bucket_name],
comparison_buckets[bucket_name])
result[bucket_name + ' diff'] = bucket_diff
@ -289,7 +288,7 @@ def _format_diff_result(dr):
return dr
def _rendered_doc(revision_id):
def _render_documents(revision_id):
"""Provides rendered document by given revision id."""
filters = {'deleted': False}
rendered_documents, _ = common.get_rendered_docs(revision_id, **filters)

View File

@ -25,12 +25,19 @@ import testtools
from deckhand.conf import config # noqa: Calls register_opts(CONF)
from deckhand.db.sqlalchemy import api as db_api
from deckhand.engine import cache
from deckhand.tests import test_utils
from deckhand.tests.unit import fixtures as dh_fixtures
CONF = cfg.CONF
logging.register_options(CONF)
logging.setup(CONF, 'deckhand')
BASE_EXPECTED_FIELDS = ("created_at", "updated_at", "deleted_at", "deleted")
DOCUMENT_EXPECTED_FIELDS = BASE_EXPECTED_FIELDS + (
"id", "schema", "name", "layer", "metadata", "data", "data_hash",
"metadata_hash", "revision_id", "bucket_id")
REVISION_EXPECTED_FIELDS = ("id", "documents", "tags")
class DeckhandTestCase(testtools.TestCase):
@ -122,3 +129,112 @@ class DeckhandWithDBTestCase(DeckhandTestCase):
group='database')
db_api.setup_db(CONF.database.connection, create_tables=True)
self.addCleanup(db_api.drop_db)
def create_documents(self, bucket_name, documents,
validation_policies=None):
if not validation_policies:
validation_policies = []
if not isinstance(documents, list):
documents = [documents]
if not isinstance(validation_policies, list):
validation_policies = [validation_policies]
docs = db_api.documents_create(
bucket_name, documents, validation_policies)
return docs
def show_document(self, **fields):
doc = db_api.document_get(**fields)
self.validate_document(actual=doc)
return doc
def create_revision(self):
# Implicitly creates a revision and returns it.
documents = [DocumentFixture.get_minimal_fixture()]
bucket_name = test_utils.rand_name('bucket')
revision_id = self.create_documents(bucket_name, documents)[0][
'revision_id']
return revision_id
def show_revision(self, revision_id):
revision = db_api.revision_get(revision_id)
self.validate_revision(revision)
return revision
def delete_revisions(self):
return db_api.revision_delete_all()
def list_revision_documents(self, revision_id, **filters):
documents = db_api.revision_documents_get(revision_id, **filters)
for document in documents:
self.validate_document(document)
return documents
def list_revisions(self):
return db_api.revision_get_all()
def rollback_revision(self, revision_id):
latest_revision = db_api.revision_get_latest()
return db_api.revision_rollback(revision_id, latest_revision)
def create_validation(self, revision_id, val_name, val_data):
return db_api.validation_create(revision_id, val_name, val_data)
def _validate_object(self, obj):
for attr in BASE_EXPECTED_FIELDS:
if attr.endswith('_at'):
self.assertThat(obj[attr], testtools.matchers.MatchesAny(
testtools.matchers.Is(None),
testtools.matchers.IsInstance(str)))
else:
self.assertIsInstance(obj[attr], bool)
def validate_document(self, actual, expected=None, is_deleted=False):
self._validate_object(actual)
# Validate that the document has all expected fields and is a dict.
expected_fields = list(DOCUMENT_EXPECTED_FIELDS)
if not is_deleted:
expected_fields.remove('deleted_at')
self.assertIsInstance(actual, dict)
for field in expected_fields:
self.assertIn(field, actual)
def validate_revision(self, revision):
self._validate_object(revision)
for attr in REVISION_EXPECTED_FIELDS:
self.assertIn(attr, revision)
# TODO(felipemonteiro): Move this into a separate module called `fixtures`.
class DocumentFixture(object):
@staticmethod
def get_minimal_fixture(**kwargs):
fixture = {
'data': {
test_utils.rand_name('key'): test_utils.rand_name('value')
},
'metadata': {
'name': test_utils.rand_name('metadata_data'),
'label': test_utils.rand_name('metadata_label'),
'layeringDefinition': {
'abstract': test_utils.rand_bool(),
'layer': test_utils.rand_name('layer')
},
'storagePolicy': test_utils.rand_name('storage_policy')
},
'schema': test_utils.rand_name('schema')}
fixture.update(kwargs)
return fixture
@staticmethod
def get_minimal_multi_fixture(count=2, **kwargs):
return [DocumentFixture.get_minimal_fixture(**kwargs)
for _ in range(count)]

View File

@ -1,136 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from testtools import matchers
from deckhand.db.sqlalchemy import api as db_api
from deckhand.tests import test_utils
from deckhand.tests.unit import base
BASE_EXPECTED_FIELDS = ("created_at", "updated_at", "deleted_at", "deleted")
DOCUMENT_EXPECTED_FIELDS = BASE_EXPECTED_FIELDS + (
"id", "schema", "name", "layer", "metadata", "data", "data_hash",
"metadata_hash", "revision_id", "bucket_id")
REVISION_EXPECTED_FIELDS = ("id", "documents", "tags")
# TODO(felipemonteiro): Move this into a separate module called `fixtures`.
class DocumentFixture(object):
@staticmethod
def get_minimal_fixture(**kwargs):
fixture = {
'data': {
test_utils.rand_name('key'): test_utils.rand_name('value')
},
'metadata': {
'name': test_utils.rand_name('metadata_data'),
'label': test_utils.rand_name('metadata_label'),
'layeringDefinition': {
'abstract': test_utils.rand_bool(),
'layer': test_utils.rand_name('layer')
},
'storagePolicy': test_utils.rand_name('storage_policy')
},
'schema': test_utils.rand_name('schema')}
fixture.update(kwargs)
return fixture
@staticmethod
def get_minimal_multi_fixture(count=2, **kwargs):
return [DocumentFixture.get_minimal_fixture(**kwargs)
for _ in range(count)]
class TestDbBase(base.DeckhandWithDBTestCase):
def create_documents(self, bucket_name, documents,
validation_policies=None):
if not validation_policies:
validation_policies = []
if not isinstance(documents, list):
documents = [documents]
if not isinstance(validation_policies, list):
validation_policies = [validation_policies]
docs = db_api.documents_create(
bucket_name, documents, validation_policies)
return docs
def show_document(self, **fields):
doc = db_api.document_get(**fields)
self.validate_document(actual=doc)
return doc
def create_revision(self):
# Implicitly creates a revision and returns it.
documents = [DocumentFixture.get_minimal_fixture()]
bucket_name = test_utils.rand_name('bucket')
revision_id = self.create_documents(bucket_name, documents)[0][
'revision_id']
return revision_id
def show_revision(self, revision_id):
revision = db_api.revision_get(revision_id)
self.validate_revision(revision)
return revision
def delete_revisions(self):
return db_api.revision_delete_all()
def list_revision_documents(self, revision_id, **filters):
documents = db_api.revision_documents_get(revision_id, **filters)
for document in documents:
self.validate_document(document)
return documents
def list_revisions(self):
return db_api.revision_get_all()
def rollback_revision(self, revision_id):
latest_revision = db_api.revision_get_latest()
return db_api.revision_rollback(revision_id, latest_revision)
def create_validation(self, revision_id, val_name, val_data):
return db_api.validation_create(revision_id, val_name, val_data)
def _validate_object(self, obj):
for attr in BASE_EXPECTED_FIELDS:
if attr.endswith('_at'):
self.assertThat(obj[attr], matchers.MatchesAny(
matchers.Is(None), matchers.IsInstance(str)))
else:
self.assertIsInstance(obj[attr], bool)
def validate_document(self, actual, expected=None, is_deleted=False):
self._validate_object(actual)
# Validate that the document has all expected fields and is a dict.
expected_fields = list(DOCUMENT_EXPECTED_FIELDS)
if not is_deleted:
expected_fields.remove('deleted_at')
self.assertIsInstance(actual, dict)
for field in expected_fields:
self.assertIn(field, actual)
def validate_revision(self, revision):
self._validate_object(revision)
for attr in REVISION_EXPECTED_FIELDS:
self.assertIn(attr, revision)

View File

@ -19,10 +19,10 @@ from deckhand.db.sqlalchemy import api as db_api
from deckhand import errors
from deckhand import factories
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestDocuments(base.TestDbBase):
class TestDocuments(base.DeckhandWithDBTestCase):
def setUp(self):
super(TestDocuments, self).setUp()

View File

@ -14,10 +14,10 @@
from deckhand import errors
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestDocumentsNegative(base.TestDbBase):
class TestDocumentsNegative(base.DeckhandWithDBTestCase):
def test_get_documents_by_revision_id_and_wrong_filters(self):
payload = base.DocumentFixture.get_minimal_fixture()

View File

@ -15,10 +15,10 @@
from deckhand import errors
from deckhand import factories
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class LayeringPoliciesBaseTest(base.TestDbBase):
class LayeringPoliciesBaseTest(base.DeckhandWithDBTestCase):
def setUp(self):
super(LayeringPoliciesBaseTest, self).setUp()

View File

@ -13,10 +13,10 @@
# limitations under the License.
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestRevisionDocumentsFiltering(base.TestDbBase):
class TestRevisionDocumentsFiltering(base.DeckhandWithDBTestCase):
def test_document_filtering_by_bucket_name(self):
document = base.DocumentFixture.get_minimal_fixture()

View File

@ -14,10 +14,10 @@
from deckhand import errors
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestRevisionRollback(base.TestDbBase):
class TestRevisionRollback(base.DeckhandWithDBTestCase):
def test_create_update_rollback(self):
# Revision 1: Create 4 documents.
@ -124,7 +124,7 @@ class TestRevisionRollback(base.TestDbBase):
self.assertEmpty(rollback_documents)
class TestRevisionRollbackNegative(base.TestDbBase):
class TestRevisionRollbackNegative(base.DeckhandWithDBTestCase):
def test_rollback_to_missing_revision_raises_exc(self):
# revision_id=1 doesn't exist yet since we start from an empty DB.

View File

@ -15,10 +15,10 @@
from deckhand.db.sqlalchemy import api as db_api
from deckhand import errors
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestRevisionTags(base.TestDbBase):
class TestRevisionTags(base.DeckhandWithDBTestCase):
def setUp(self):
super(TestRevisionTags, self).setUp()

View File

@ -14,10 +14,10 @@
from deckhand.db.sqlalchemy import api as db_api
from deckhand import errors
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestRevisionTagsNegative(base.TestDbBase):
class TestRevisionTagsNegative(base.DeckhandWithDBTestCase):
def test_create_tag_revision_not_found(self):
self.assertRaises(

View File

@ -14,10 +14,10 @@
from deckhand import errors
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestRevisions(base.TestDbBase):
class TestRevisions(base.DeckhandWithDBTestCase):
def test_list(self):
documents = [base.DocumentFixture.get_minimal_fixture()

View File

@ -16,10 +16,10 @@ import copy
from deckhand.engine import revision_diff
from deckhand import factories
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestRevisionDeepDiffing(base.TestDbBase):
class TestRevisionDeepDiffing(base.DeckhandWithDBTestCase):
def _test_data(self):
return {

View File

@ -16,10 +16,10 @@ import copy
from deckhand.engine.revision_diff import revision_diff
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestRevisionDiffing(base.TestDbBase):
class TestRevisionDiffing(base.DeckhandWithDBTestCase):
def _verify_buckets_status(self, revision_id, comparison_revision_id,
expected):
@ -307,3 +307,30 @@ class TestRevisionDiffing(base.TestDbBase):
self._verify_buckets_status(
revision_id_1, revision_id_4,
{bucket_name: 'unmodified', alt_bucket_name_2: 'created'})
def test_revision_diff_delete_then_rollback(self):
"""Validate that rolling back a revision works with bucket diff."""
payload = base.DocumentFixture.get_minimal_fixture()
bucket_name = test_utils.rand_name('bucket')
created_documents = self.create_documents(bucket_name, payload)
revision_id = created_documents[0]['revision_id']
# Delete all previously created documents.
deleted_documents = self.create_documents(bucket_name, [])
comparison_revision_id = deleted_documents[0]['revision_id']
# Validate that the empty bucket is deleted.
self._verify_buckets_status(
revision_id, comparison_revision_id, {bucket_name: 'deleted'})
# Rollback to first non-empty revision.
rollback_revision_id = self.rollback_revision(revision_id)['id']
# Validate that diffing rolled-back revision against 1 is unmodified.
self._verify_buckets_status(
revision_id, rollback_revision_id, {bucket_name: 'unmodified'})
# Validate that diffing rolled-back revision against 2 is created
# (because the rolled-back revision is newer than revision 2).
self._verify_buckets_status(
comparison_revision_id, rollback_revision_id,
{bucket_name: 'created'})

View File

@ -27,10 +27,10 @@ from deckhand.engine import secrets_manager
from deckhand import errors
from deckhand import factories
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base as test_base
from deckhand.tests.unit import base as test_base
class TestSecretsManager(test_base.TestDbBase):
class TestSecretsManager(test_base.DeckhandWithDBTestCase):
def setUp(self):
super(TestSecretsManager, self).setUp()
@ -168,7 +168,7 @@ class TestSecretsManager(test_base.TestDbBase):
self.assertEqual(payload, retrieved_payload)
class TestSecretsSubstitution(test_base.TestDbBase):
class TestSecretsSubstitution(test_base.DeckhandWithDBTestCase):
def setUp(self):
super(TestSecretsSubstitution, self).setUp()
@ -876,7 +876,7 @@ data:
self.assertEqual(expected, substituted_docs[0])
class TestSecretsSubstitutionNegative(test_base.TestDbBase):
class TestSecretsSubstitutionNegative(test_base.DeckhandWithDBTestCase):
def setUp(self):
super(TestSecretsSubstitutionNegative, self).setUp()

View File

@ -14,10 +14,10 @@
from deckhand.control.views import document
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestDocumentViews(base.TestDbBase):
class TestDocumentViews(base.DeckhandWithDBTestCase):
def setUp(self):
super(TestDocumentViews, self).setUp()

View File

@ -15,10 +15,10 @@
from deckhand.control.views import revision_tag
from deckhand.db.sqlalchemy import api as db_api
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestRevisionViews(base.TestDbBase):
class TestRevisionViews(base.DeckhandWithDBTestCase):
def setUp(self):
super(TestRevisionViews, self).setUp()

View File

@ -14,10 +14,10 @@
from deckhand.control.views import revision
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit import base
class TestRevisionViews(base.TestDbBase):
class TestRevisionViews(base.DeckhandWithDBTestCase):
def setUp(self):
super(TestRevisionViews, self).setUp()