From bead00e23e6ebba1b1c859206590592d2792d9bb Mon Sep 17 00:00:00 2001
From: Felipe Monteiro <felipe.monteiro@att.com>
Date: Wed, 25 Oct 2017 23:12:13 +0100
Subject: [PATCH] HTML test report for Deckhand functional tests

This PS leverages pytest-html to generate an HTML report for
Deckhand functional tests. The test_gabbi.py file was updated to
use pytest as the test runner as it is compatible with pytest-html.
The report is saved in results/index.html. Test docs were updated.

Change-Id: I0b611bf124bf87d801ab93dd2a5d16f136e4801d
---
 .gitignore                              |  1 +
 deckhand/tests/functional/test_gabbi.py | 31 +++++++++++++------------
 doc/source/testing.rst                  |  2 ++
 test-requirements.txt                   |  1 +
 tools/functional-tests.sh               |  6 +++--
 5 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/.gitignore b/.gitignore
index dee2eb72..558c70b6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,6 +47,7 @@ coverage.xml
 .hypothesis/
 .testrepository/*
 cover/*
+results/*
 
 # Translations
 *.mo
diff --git a/deckhand/tests/functional/test_gabbi.py b/deckhand/tests/functional/test_gabbi.py
index 810c6775..871d682e 100644
--- a/deckhand/tests/functional/test_gabbi.py
+++ b/deckhand/tests/functional/test_gabbi.py
@@ -15,16 +15,16 @@
 import os
 import yaml
 
-import gabbi.driver
-import gabbi.handlers.jsonhandler
-import gabbi.json_parser
+from gabbi import driver
+from gabbi.driver import test_pytest  # noqa
+from gabbi.handlers import jsonhandler
 
 TESTS_DIR = 'gabbits'
 
 
 # This is quite similar to the existing JSONHandler, so use it as the base
-# class instead of gabbi.handlers.base.ContentHandler
-class MultidocJsonpaths(gabbi.handlers.jsonhandler.JSONHandler):
+# class instead of `gabbi.handlers.base.ContentHandler`.
+class MultidocJsonpaths(jsonhandler.JSONHandler):
     test_key_suffix = 'multidoc_jsonpaths'
 
     @staticmethod
@@ -46,14 +46,15 @@ class MultidocJsonpaths(gabbi.handlers.jsonhandler.JSONHandler):
         return list(yaml.safe_load_all(string))
 
 
-def load_tests(loader, tests, pattern):
+def pytest_generate_tests(metafunc):
     test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
-    return gabbi.driver.build_tests(test_dir, loader,
-        # NOTE(fmontei): When there are multiple handlers listed that
-        # accept the same content-type, the one that is earliest in the
-        # list will be used. Thus, we cannot specify multiple content
-        # handlers for handling list/dictionary responses from the server
-        # using different handlers.
-        content_handlers=[MultidocJsonpaths],
-        verbose=True,
-        url=os.environ['DECKHAND_TEST_URL'])
+    # NOTE(fmontei): While only `url` or `host` is needed, strangely both
+    # are needed because we use `pytest-html` which throws an error without
+    # `host`.
+    driver.py_test_generator(
+        test_dir, url=os.environ['DECKHAND_TEST_URL'], host='localhost',
+        # NOTE(fmontei): When there are multiple handlers listed that accept
+        # the same content-type, the one that is earliest in the list will be
+        # used. Thus, we cannot specify multiple content handlers for handling
+        # list/dictionary responses from the server using different handlers.
+        content_handlers=[MultidocJsonpaths], metafunc=metafunc)
diff --git a/doc/source/testing.rst b/doc/source/testing.rst
index cff0367b..29991d66 100644
--- a/doc/source/testing.rst
+++ b/doc/source/testing.rst
@@ -90,6 +90,8 @@ The command executes ``tools/functional-tests.sh`` which:
        Keystone and pass an admin token to Deckhand.
     4) Instantiates Deckhand via ``uwisgi``.
     5) Calls gabbi which runs a battery of functional tests.
+    6) An HTML report that visualizes the result of the test run is output to
+       ``results/index.html``.
 
 At this time, there are no functional tests for policy enforcement
 verification. Negative tests will be added at a later date to confirm that
diff --git a/test-requirements.txt b/test-requirements.txt
index 4a11f88f..9d540674 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -19,3 +19,4 @@ gabbi==1.35.1
 sphinx_rtd_theme==0.2.4
 pifpaf==0.10.0
 reno>=2.5.0 # Apache-2.0
+pytest-html==1.16.0
diff --git a/tools/functional-tests.sh b/tools/functional-tests.sh
index 34fb7bd3..65558775 100755
--- a/tools/functional-tests.sh
+++ b/tools/functional-tests.sh
@@ -127,12 +127,14 @@ sleep 5
 
 log_section Running tests
 
+ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
 set +e
 posargs=$@
 if [ ${#posargs} -ge 1 ]; then
-    ostestr --concurrency 1 --regex $1
+    py.test -k $1 -svx $( dirname $ROOTDIR )/deckhand/tests/functional/test_gabbi.py --html=results/index.html
 else
-    ostestr --concurrency 1
+    py.test -svx $( dirname $ROOTDIR )/deckhand/tests/functional/test_gabbi.py --html=results/index.html
 fi
 TEST_STATUS=$?
 set -e