[WIP] Implement intermediary file validation

Change-Id: Iaa385d265b027426f8e5f2376462ffb4c0d1d3fa

Pipfile.lock

@@ -128,7 +128,7 @@
         },
         "spyglass-plugin-xls": {
             "git": "https://opendev.org/airship/spyglass-plugin-xls.git",
-            "ref": "269ce7154f88a51e5dd9b883e2f7f15d5326a905"
+            "ref": "3b794d05ffd4731e1b7c4f23bc73a3d73f5ba1c1"
         }
     },
     "develop": {
@@ -198,13 +198,6 @@
             ],
             "version": "==4.5.4"
         },
-        "ddt": {
-            "hashes": [
-                "sha256:474546b4020ce8a2f9550ba8899c28aa2c284c7bbf175bddede98be949d1ca7c",
-                "sha256:d13e6af8f36238e89d00f4ebccf2bda4f6d1878be560a6600689e42077e164e3"
-            ],
-            "version": "==1.2.1"
-        },
         "execnet": {
             "hashes": [
                 "sha256:0dd40ad3b960aae93bdad7fe1c3f049bbcc8fba47094655a4301f5b33e906816",
@@ -226,12 +219,6 @@
             ],
             "version": "==2.6.2"
         },
-        "gitdb": {
-            "hashes": [
-                "sha256:a3ebbc27be035a2e874ed904df516e35f4a29a778a764385de09de9e0f139658"
-            ],
-            "version": "==0.6.4"
-        },
         "gitdb2": {
             "hashes": [
                 "sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2",
@@ -241,10 +228,10 @@
         },
         "gitpython": {
             "hashes": [
-                "sha256:556b64796c5e268b35e3b431a429e813ad54aa178e1baaec2a9ba82e8575a89e",
-                "sha256:629867ebf609cef21bb9d849039e281e25963fb7d714a2f6bacc1ecce4800293"
+                "sha256:947cc75913e7b6da108458136607e2ee0e40c20be1e12d4284e7c6c12956c276",
+                "sha256:d2f4945f8260f6981d724f5957bc076398ada55cb5d25aaee10108bcdc894100"
             ],
-            "version": "==3.0.0"
+            "version": "==3.0.2"
         },
         "hacking": {
             "hashes": [
@@ -259,6 +246,7 @@
                 "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8",
                 "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3"
             ],
+            "markers": "python_version < '3.8'",
             "version": "==0.19"
         },
         "mccabe": {
@@ -327,11 +315,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:6ef6d06de77ce2961156013e9dff62f1b2688aa04d0dc244299fe7d67e09370d",
-                "sha256:a736fed91c12681a7b34617c8fcefe39ea04599ca72c608751c31d89579a3f77"
+                "sha256:95b1f6db806e5b1b5b443efeb58984c24945508f93a866c1719e1a507a957d7c",
+                "sha256:c3d5020755f70c82eceda3feaf556af9a341334414a8eca521a18f463bcead88"
             ],
             "index": "pypi",
-            "version": "==5.0.1"
+            "version": "==5.1.1"
         },
         "pytest-cov": {
             "hashes": [
@@ -380,12 +368,6 @@
             ],
             "version": "==1.12.0"
         },
-        "smmap": {
-            "hashes": [
-                "sha256:0e2b62b497bd5f0afebc002eda4d90df9d209c30ef257e8673c90a6b5c119d62"
-            ],
-            "version": "==0.9.0"
-        },
         "smmap2": {
             "hashes": [
                 "sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde",
@@ -417,10 +399,10 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:6cb2e4c18d22dbbe283d0a0c31bb7d90771a606b2cb3415323eea008eaee6a9d",
-                "sha256:909fe0d3f7c9151b2df0a2cb53e55bdb7b0d61469353ff7a49fd47b0f0ab9285"
+                "sha256:94a6898293d07f84a98add34c4df900f8ec64a570292279f6d91c781d37fd305",
+                "sha256:f6fc312c031f2d2344f885de114f1cb029dfcffd26aa6e57d2ee2296935c4e7d"
             ],
-            "version": "==16.7.2"
+            "version": "==16.7.4"
         },
         "wcwidth": {
             "hashes": [
@@ -439,10 +421,10 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a",
-                "sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec"
+                "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e",
+                "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"
             ],
-            "version": "==0.5.2"
+            "version": "==0.6.0"
         }
     }
 }

spyglass/cli.py

@@ -79,7 +79,21 @@ FORCE_OPTION = click.option(
     'force',
     is_flag=True,
     default=False,
-    help="Forces manifests to be written, regardless of undefined data.")
+    help='Forces manifests to be written, regardless of undefined data.')
+
+INTERMEDIARY_SCHEMA_OPTION = click.option(
+    '--intermediary-schema',
+    'intermediary_schema',
+    type=click.Path(exists=True, readable=True, dir_okay=False),
+    default=pkg_resources.resource_filename('spyglass', "schemas/intermediary_schema.json"),
+    help='Path to the intermediary schema to be used for validation.')
+
+NO_INTERMEDIARY_VALIDATION_OPTION = click.option(
+    '--no-validation',
+    'no_validation',
+    is_flag=True,
+    default=False,
+    help='Skips validation on generated intermediary data.')
 
 
 @click.option(
@@ -135,7 +149,9 @@ def intermediary_processor(plugin_type, **kwargs):
     # Apply design rules to the data
     LOG.info("Apply design rules to the extracted data")
     process_input_ob = ProcessDataSource(
-        kwargs['site_name'], data_extractor.data)
+        kwargs['site_name'], data_extractor.data,
+        kwargs.get('intermediary_schema', None),
+        kwargs.get('no_validation', False))
     return process_input_ob
 
 

spyglass/parser/engine.py

@@ -20,6 +20,7 @@ import pprint
 import sys
 
 import jsonschema
+from jsonschema import Draft7Validator
 from netaddr import IPNetwork
 from pkg_resources import resource_filename
 import yaml
@@ -28,13 +29,16 @@ LOG = logging.getLogger(__name__)
 
 
 class ProcessDataSource(object):
-    def __init__(self, region, extracted_data):
+    def __init__(self, region, extracted_data, intermediary_schema, no_validation):
         # Initialize intermediary and save site type
         self.host_type = {}
         self.sitetype = None
         self.genesis_node = None
         self.network_subnets = None
         self.region_name = region
+        with open(intermediary_schema, 'r') as loaded_schema:
+            self.intermediary_schema = json.load(loaded_schema)
+        self.no_validation = no_validation
 
         LOG.info("Loading plugin data source")
         self.data = extracted_data
@@ -74,17 +78,17 @@ class ProcessDataSource(object):
             "Genesis Node Details:\n{}".format(
                 pprint.pformat(self.genesis_node)))
 
-    @staticmethod
-    def _validate_intermediary_data(data):
+    def _validate_intermediary_data(self, data):
         """Validates the intermediary data before generating manifests.
 
         It checks whether the data types and data format are as expected.
         The method validates this with regex pattern defined for each
         data type.
         """
-        # TODO(ian-pittwood): Implement intermediary validation or remove
 
         LOG.info("Validating Intermediary data")
+        validator = Draft7Validator(self.intermediary_schema)
+        errors = sorted(validator.iter_errors(data), key=lambda e: e.path)
         # Performing a deep copy
         temp_data = copy.deepcopy(data)
         # Converting baremetal dict to list.
@@ -97,28 +101,6 @@ class ProcessDataSource(object):
         schema_dir = resource_filename("spyglass", "schemas/")
         schema_file = schema_dir + "data_schema.json"
         json_data = json.loads(json.dumps(temp_data))
-        with open(schema_file, "r") as f:
-            json_schema = json.load(f)
-        try:
-            # Suppressing writing of data2.json. Can use it for debugging
-            # with open('data2.json', 'w') as outfile:
-            #     json.dump(temp_data, outfile, sort_keys=True, indent=4)
-            jsonschema.validate(json_data, json_schema)
-        except jsonschema.exceptions.ValidationError as e:
-            LOG.error("Validation Error")
-            LOG.error("Message:{}".format(e.message))
-            LOG.error("Validator_path:{}".format(e.path))
-            LOG.error("Validator_pattern:{}".format(e.validator_value))
-            LOG.error("Validator:{}".format(e.validator))
-            sys.exit()
-        except jsonschema.exceptions.SchemaError as e:
-            LOG.error("Schema Validation Error!!")
-            LOG.error("Message:{}".format(e.message))
-            LOG.error("Schema:{}".format(e.schema))
-            LOG.error("Validator_value:{}".format(e.validator_value))
-            LOG.error("Validator:{}".format(e.validator))
-            LOG.error("path:{}".format(e.path))
-            sys.exit()
 
         LOG.info("Data validation Passed!")