diff --git a/deckhand/tests/functional/README.rst b/deckhand/tests/functional/README.rst new file mode 100644 index 00000000..2bd329f2 --- /dev/null +++ b/deckhand/tests/functional/README.rst @@ -0,0 +1,30 @@ +Functional Tests +================ + +Deckhand uses `gabbi`_ to drive its functional tests. The entry point for +these tests is ``functional-tests.sh`` under ``tools`` directory. + +Directory Test Layout +--------------------- + +Tests are contained in intuitively named subdirectories nested under +``deckhand/tests/functional/gabbits``. For example, layering tests are +contained under the ``layering`` subdirectory. This pattern should be strictly +followed. + +Because `gabbi`_ does not support loading tests from subdirectories, logic +is included in ``test_gabbi.py`` to: + +#. Create a temporary directory. +#. Create a symlink between all the test files in the nested subdirectories + and the temporary directory. + +However, the test directory can still be modified: + +* New subdirectories under ``gabbits`` can be added. +* New tests under any of those subdirectories can be added. +* New resource files under ``gabits/resources`` can be added. This directory + name should never be renamed. +* All other subdirectories, test files, and resources may be renamed. + +.. _gabbi: https://gabbi.readthedocs.io/en/latest/gabbi.html diff --git a/deckhand/tests/functional/gabbits/document-crud-error-bucket-conflict.yaml b/deckhand/tests/functional/gabbits/document/document-crud-error-bucket-conflict.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-crud-error-bucket-conflict.yaml rename to deckhand/tests/functional/gabbits/document/document-crud-error-bucket-conflict.yaml diff --git a/deckhand/tests/functional/gabbits/document-crud-success-multi-bucket.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-multi-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-crud-success-multi-bucket.yaml rename to deckhand/tests/functional/gabbits/document/document-crud-success-multi-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-crud-success-owned-documents.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-crud-success-owned-documents.yaml rename to deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml diff --git a/deckhand/tests/functional/gabbits/document-crud-success-single-bucket.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-crud-success-single-bucket.yaml rename to deckhand/tests/functional/gabbits/document/document-crud-success-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-crud-success-unusual-documents.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-crud-success-unusual-documents.yaml rename to deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-layering.yaml b/deckhand/tests/functional/gabbits/layering/layering-multiple-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-layering.yaml rename to deckhand/tests/functional/gabbits/layering/layering-multiple-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-error-layering-policy-conflict.yaml b/deckhand/tests/functional/gabbits/layering/layering-policy-conflict.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-error-layering-policy-conflict.yaml rename to deckhand/tests/functional/gabbits/layering/layering-policy-conflict.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-layering.yaml b/deckhand/tests/functional/gabbits/layering/layering-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-layering.yaml rename to deckhand/tests/functional/gabbits/layering/layering-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-replacement.yaml b/deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-replacement.yaml rename to deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-layering-and-substitution.yaml b/deckhand/tests/functional/gabbits/layering/layering-with-substitution-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-layering-and-substitution.yaml rename to deckhand/tests/functional/gabbits/layering/layering-with-substitution-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/revision-diff-success.yaml b/deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-diff-success.yaml rename to deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml diff --git a/deckhand/tests/functional/gabbits/revision-documents-filters-negative.yaml b/deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters-negative.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-documents-filters-negative.yaml rename to deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters-negative.yaml diff --git a/deckhand/tests/functional/gabbits/revision-documents-filters.yaml b/deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-documents-filters.yaml rename to deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters.yaml diff --git a/deckhand/tests/functional/gabbits/revision-documents-multiple-filters.yaml b/deckhand/tests/functional/gabbits/revision-documents/revision-documents-multiple-filters.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-documents-multiple-filters.yaml rename to deckhand/tests/functional/gabbits/revision-documents/revision-documents-multiple-filters.yaml diff --git a/deckhand/tests/functional/gabbits/rollback-success-single-bucket.yaml b/deckhand/tests/functional/gabbits/revision-rollback/rollback-success-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/rollback-success-single-bucket.yaml rename to deckhand/tests/functional/gabbits/revision-rollback/rollback-success-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/revision-tag-success.yaml b/deckhand/tests/functional/gabbits/revision-tag/revision-tag-success.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-tag-success.yaml rename to deckhand/tests/functional/gabbits/revision-tag/revision-tag-success.yaml diff --git a/deckhand/tests/functional/gabbits/revision-crud-success-single-bucket.yaml b/deckhand/tests/functional/gabbits/revision/revision-crud-success-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-crud-success-single-bucket.yaml rename to deckhand/tests/functional/gabbits/revision/revision-crud-success-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/revision-filters.yaml b/deckhand/tests/functional/gabbits/revision/revision-filters.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/revision-filters.yaml rename to deckhand/tests/functional/gabbits/revision/revision-filters.yaml diff --git a/deckhand/tests/functional/gabbits/schema-validation-success-ucp-sample.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/schema-validation-success-ucp-sample.yaml rename to deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml diff --git a/deckhand/tests/functional/gabbits/schema-validation-success.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/schema-validation-success.yaml rename to deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml diff --git a/deckhand/tests/functional/gabbits/schema-validation-with-validation-policy-success.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/schema-validation-with-validation-policy-success.yaml rename to deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml diff --git a/deckhand/tests/functional/gabbits/chained-substitution.yaml b/deckhand/tests/functional/gabbits/substitution/substitution-chained-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/chained-substitution.yaml rename to deckhand/tests/functional/gabbits/substitution/substitution-chained-single-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml b/deckhand/tests/functional/gabbits/substitution/substitution-multiple-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml rename to deckhand/tests/functional/gabbits/substitution/substitution-multiple-bucket.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution-generic.yaml b/deckhand/tests/functional/gabbits/substitution/substitution-single-bucket-generic.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution-generic.yaml rename to deckhand/tests/functional/gabbits/substitution/substitution-single-bucket-generic.yaml diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml b/deckhand/tests/functional/gabbits/substitution/substitution-single-bucket.yaml similarity index 100% rename from deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml rename to deckhand/tests/functional/gabbits/substitution/substitution-single-bucket.yaml diff --git a/deckhand/tests/functional/test_gabbi.py b/deckhand/tests/functional/test_gabbi.py index 871d682e..465e8fcc 100644 --- a/deckhand/tests/functional/test_gabbi.py +++ b/deckhand/tests/functional/test_gabbi.py @@ -12,14 +12,55 @@ # See the License for the specific language governing permissions and # limitations under the License. +import atexit import os +import shutil +import tempfile import yaml from gabbi import driver from gabbi.driver import test_pytest # noqa from gabbi.handlers import jsonhandler -TESTS_DIR = 'gabbits' +TEST_DIR = tempfile.mkdtemp(prefix='deckhand') + + +def __create_temp_test_dir(): + """Hack around the fact that gabbi doesn't support loading tests contained + in subdirectories. This inconvenience leads to poor test directory layout + in which all the test files are contained in one directory. + + """ + root_test_dir = os.path.join(os.path.dirname(__file__), 'gabbits') + test_files = [] + + for root, dirs, files in os.walk(root_test_dir): + is_test_file = ( + 'gabbits' in root and not root.endswith('gabbits') + ) + if is_test_file: + test_files.extend([os.path.abspath(os.path.join(root, f)) + for f in files]) + + resources_dir = os.path.join(TEST_DIR, 'resources') + if not os.path.exists(resources_dir): + os.makedirs(resources_dir) + + for test_file in test_files: + basename = os.path.basename(test_file) + if 'resources' in test_file: + os.symlink(test_file, os.path.join(resources_dir, basename)) + else: + os.symlink(test_file, os.path.join(TEST_DIR, basename)) + + +__create_temp_test_dir() + + +@atexit.register +def __remove_temp_test_dir(): + if os.path.exists(TEST_DIR): + shutil.rmtree(TEST_DIR) # This is quite similar to the existing JSONHandler, so use it as the base @@ -47,12 +88,11 @@ class MultidocJsonpaths(jsonhandler.JSONHandler): def pytest_generate_tests(metafunc): - test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR) # NOTE(fmontei): While only `url` or `host` is needed, strangely both # are needed because we use `pytest-html` which throws an error without # `host`. driver.py_test_generator( - test_dir, url=os.environ['DECKHAND_TEST_URL'], host='localhost', + TEST_DIR, url=os.environ['DECKHAND_TEST_URL'], host='localhost', # NOTE(fmontei): When there are multiple handlers listed that accept # the same content-type, the one that is earliest in the list will be # used. Thus, we cannot specify multiple content handlers for handling diff --git a/tools/common-tests.sh b/tools/common-tests.sh index 4c17103b..0de0cf0c 100644 --- a/tools/common-tests.sh +++ b/tools/common-tests.sh @@ -154,6 +154,7 @@ function gen_paste { local disable_keystone=$1 if $disable_keystone; then + log_section Disabling Keystone authentication. sed 's/authtoken api/api/' etc/deckhand/deckhand-paste.ini &> $CONF_DIR/deckhand-paste.ini else cp etc/deckhand/deckhand-paste.ini $CONF_DIR/deckhand-paste.ini diff --git a/tools/functional-tests.sh b/tools/functional-tests.sh index 5755b6ef..f1d3cacb 100755 --- a/tools/functional-tests.sh +++ b/tools/functional-tests.sh @@ -51,7 +51,7 @@ function deploy_deckhand { gen_policy if [ -z "$DECKHAND_IMAGE" ]; then - log_section "Running Deckhand via uwsgi" + log_section "Running Deckhand via uwsgi." alembic upgrade head # NOTE(fmontei): Deckhand's database is not configured to work with @@ -63,7 +63,7 @@ function deploy_deckhand { export DECKHAND_API_THREADS=4 source $ROOTDIR/../entrypoint.sh server & else - log_section "Running Deckhand via Docker" + log_section "Running Deckhand via Docker." sudo docker run \ --rm \ --net=host \