From 0bed580daa47b89c9ce2daa87653d058cb92b4f4 Mon Sep 17 00:00:00 2001 From: Ian H Pittwood Date: Mon, 26 Aug 2019 14:31:33 -0500 Subject: [PATCH] Implement intermediary file validation Implements JSON schema validation for intermediary YAML files Adds tests for intermediary validation Change-Id: Iaa385d265b027426f8e5f2376462ffb4c0d1d3fa --- Pipfile.lock | 46 +-- spyglass/cli.py | 21 +- spyglass/exceptions.py | 27 +- spyglass/parser/engine.py | 65 ++- ...a_schema.json => intermediary_schema.json} | 2 +- tests/conftest.py | 9 + tests/shared/intermediary_schema.json | 373 ++++++++++++++++++ tests/shared/invalid_intermediary.yaml | 216 ++++++++++ tests/unit/parser/test_engine.py | 17 +- 9 files changed, 688 insertions(+), 88 deletions(-) rename spyglass/schemas/{data_schema.json => intermediary_schema.json} (99%) create mode 100644 tests/shared/intermediary_schema.json create mode 100644 tests/shared/invalid_intermediary.yaml diff --git a/Pipfile.lock b/Pipfile.lock index 3dd3a7d..7cd9f61 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -128,7 +128,7 @@ }, "spyglass-plugin-xls": { "git": "https://opendev.org/airship/spyglass-plugin-xls.git", - "ref": "269ce7154f88a51e5dd9b883e2f7f15d5326a905" + "ref": "3b794d05ffd4731e1b7c4f23bc73a3d73f5ba1c1" } }, "develop": { @@ -198,13 +198,6 @@ ], "version": "==4.5.4" }, - "ddt": { - "hashes": [ - "sha256:474546b4020ce8a2f9550ba8899c28aa2c284c7bbf175bddede98be949d1ca7c", - "sha256:d13e6af8f36238e89d00f4ebccf2bda4f6d1878be560a6600689e42077e164e3" - ], - "version": "==1.2.1" - }, "execnet": { "hashes": [ "sha256:0dd40ad3b960aae93bdad7fe1c3f049bbcc8fba47094655a4301f5b33e906816", @@ -226,12 +219,6 @@ ], "version": "==2.6.2" }, - "gitdb": { - "hashes": [ - "sha256:a3ebbc27be035a2e874ed904df516e35f4a29a778a764385de09de9e0f139658" - ], - "version": "==0.6.4" - }, "gitdb2": { "hashes": [ "sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2", @@ -241,10 +228,10 @@ }, "gitpython": { "hashes": [ - "sha256:556b64796c5e268b35e3b431a429e813ad54aa178e1baaec2a9ba82e8575a89e", - "sha256:629867ebf609cef21bb9d849039e281e25963fb7d714a2f6bacc1ecce4800293" + "sha256:947cc75913e7b6da108458136607e2ee0e40c20be1e12d4284e7c6c12956c276", + "sha256:d2f4945f8260f6981d724f5957bc076398ada55cb5d25aaee10108bcdc894100" ], - "version": "==3.0.0" + "version": "==3.0.2" }, "hacking": { "hashes": [ @@ -259,6 +246,7 @@ "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8", "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3" ], + "markers": "python_version < '3.8'", "version": "==0.19" }, "mccabe": { @@ -327,11 +315,11 @@ }, "pytest": { "hashes": [ - "sha256:6ef6d06de77ce2961156013e9dff62f1b2688aa04d0dc244299fe7d67e09370d", - "sha256:a736fed91c12681a7b34617c8fcefe39ea04599ca72c608751c31d89579a3f77" + "sha256:95b1f6db806e5b1b5b443efeb58984c24945508f93a866c1719e1a507a957d7c", + "sha256:c3d5020755f70c82eceda3feaf556af9a341334414a8eca521a18f463bcead88" ], "index": "pypi", - "version": "==5.0.1" + "version": "==5.1.1" }, "pytest-cov": { "hashes": [ @@ -380,12 +368,6 @@ ], "version": "==1.12.0" }, - "smmap": { - "hashes": [ - "sha256:0e2b62b497bd5f0afebc002eda4d90df9d209c30ef257e8673c90a6b5c119d62" - ], - "version": "==0.9.0" - }, "smmap2": { "hashes": [ "sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde", @@ -417,10 +399,10 @@ }, "virtualenv": { "hashes": [ - "sha256:6cb2e4c18d22dbbe283d0a0c31bb7d90771a606b2cb3415323eea008eaee6a9d", - "sha256:909fe0d3f7c9151b2df0a2cb53e55bdb7b0d61469353ff7a49fd47b0f0ab9285" + "sha256:94a6898293d07f84a98add34c4df900f8ec64a570292279f6d91c781d37fd305", + "sha256:f6fc312c031f2d2344f885de114f1cb029dfcffd26aa6e57d2ee2296935c4e7d" ], - "version": "==16.7.2" + "version": "==16.7.4" }, "wcwidth": { "hashes": [ @@ -439,10 +421,10 @@ }, "zipp": { "hashes": [ - "sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a", - "sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec" + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" ], - "version": "==0.5.2" + "version": "==0.6.0" } } } diff --git a/spyglass/cli.py b/spyglass/cli.py index fb1c5fc..b16ea3e 100644 --- a/spyglass/cli.py +++ b/spyglass/cli.py @@ -79,7 +79,22 @@ FORCE_OPTION = click.option( 'force', is_flag=True, default=False, - help="Forces manifests to be written, regardless of undefined data.") + help='Forces manifests to be written, regardless of undefined data.') + +INTERMEDIARY_SCHEMA_OPTION = click.option( + '--intermediary-schema', + 'intermediary_schema', + type=click.Path(exists=True, readable=True, dir_okay=False), + default=pkg_resources.resource_filename( + 'spyglass', "schemas/intermediary_schema.json"), + help='Path to the intermediary schema to be used for validation.') + +NO_INTERMEDIARY_VALIDATION_OPTION = click.option( + '--no-validation', + 'no_validation', + is_flag=True, + default=False, + help='Skips validation on generated intermediary data.') @click.option( @@ -135,7 +150,9 @@ def intermediary_processor(plugin_type, **kwargs): # Apply design rules to the data LOG.info("Apply design rules to the extracted data") process_input_ob = ProcessDataSource( - kwargs['site_name'], data_extractor.data) + kwargs['site_name'], data_extractor.data, + kwargs.get('intermediary_schema', None), + kwargs.get('no_validation', False)) return process_input_ob diff --git a/spyglass/exceptions.py b/spyglass/exceptions.py index 9a87a8c..0af4973 100644 --- a/spyglass/exceptions.py +++ b/spyglass/exceptions.py @@ -34,8 +34,8 @@ class SpyglassBaseException(Exception): class UnsupportedPlugin(SpyglassBaseException): """Exception that occurs when a plugin is called that does not exist - :param plugin_name: name of the specified plugin - :param entry_point: the package used to access plugin_name + :keyword plugin_name: name of the specified plugin + :keyword entry_point: the package used to access plugin_name """ message = ( '%(plugin_name) was not found in the package %(entry_point) ' @@ -45,10 +45,19 @@ class UnsupportedPlugin(SpyglassBaseException): # Data Extractor exceptions +class IntermediaryValidationException(SpyglassBaseException): + """Exception that occurs when the generated intermediary fails validation + + :keyword errors: list of errors generated by the jsonschema validator + """ + message = ( + 'Intermediary validation failed with the following errors: %(errors)') + + class InvalidIntermediary(SpyglassBaseException): """Exception that occurs when data is missing from the intermediary file - :param key: dictionary key that Spyglass attempted to access + :keyword key: dictionary key that Spyglass attempted to access """ message = '%(key) is not defined in the given intermediary file.' @@ -59,8 +68,8 @@ class InvalidIntermediary(SpyglassBaseException): class PathDoesNotExistError(SpyglassBaseException): """Exception that occurs when the document or schema path does not exist - :param file_type: type of the files being accessed, documents or schemas - :param path: nonexistent path attempted to access + :keyword file_type: type of the files being accessed, documents or schemas + :keyword path: nonexistent path attempted to access """ message = '%(file_type) path: %(path) does not exist.' @@ -68,8 +77,8 @@ class PathDoesNotExistError(SpyglassBaseException): class UnexpectedFileType(SpyglassBaseException): """Exception that occurs when an unexpected file type is given - :param found_ext: the extension of the file given - :param expected_ext: the extension that was expected for the file + :keyword found_ext: the extension of the file given + :keyword expected_ext: the extension that was expected for the file """ message = ( 'Unexpected file type %(found_ext), ' @@ -82,7 +91,7 @@ class DirectoryEmptyError(SpyglassBaseException): This exception can occur when either a directory is empty or if a directory does not have any files with the correct file extension. - :param ext: file extension being searched for - :param path: path being searched for files of the specified extension + :keyword ext: file extension being searched for + :keyword path: path being searched for files of the specified extension """ message = 'No files with %(ext) extension found in document path %(path)' diff --git a/spyglass/parser/engine.py b/spyglass/parser/engine.py index f869502..68df990 100755 --- a/spyglass/parser/engine.py +++ b/spyglass/parser/engine.py @@ -12,29 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy import json import logging import os import pprint -import sys -import jsonschema +from jsonschema import Draft7Validator from netaddr import IPNetwork from pkg_resources import resource_filename import yaml +from spyglass import exceptions + LOG = logging.getLogger(__name__) class ProcessDataSource(object): - def __init__(self, region, extracted_data): + def __init__( + self, + region, + extracted_data, + intermediary_schema=None, + no_validation=True): # Initialize intermediary and save site type self.host_type = {} self.sitetype = None self.genesis_node = None self.network_subnets = None self.region_name = region + self.no_validation = no_validation + if intermediary_schema and not self.no_validation: + with open(intermediary_schema, 'r') as loaded_schema: + self.intermediary_schema = json.load(loaded_schema) LOG.info("Loading plugin data source") self.data = extracted_data @@ -74,53 +83,24 @@ class ProcessDataSource(object): "Genesis Node Details:\n{}".format( pprint.pformat(self.genesis_node))) - @staticmethod - def _validate_intermediary_data(data): + def _validate_intermediary_data(self): """Validates the intermediary data before generating manifests. It checks whether the data types and data format are as expected. The method validates this with regex pattern defined for each data type. """ - # TODO(ian-pittwood): Implement intermediary validation or remove LOG.info("Validating Intermediary data") - # Performing a deep copy - temp_data = copy.deepcopy(data) - # Converting baremetal dict to list. - baremetal_list = [] - for rack in temp_data.baremetal: - temp = [{k: v} for k, v in temp_data["baremetal"][rack].items()] - baremetal_list = baremetal_list + temp - - temp_data["baremetal"] = baremetal_list - schema_dir = resource_filename("spyglass", "schemas/") - schema_file = schema_dir + "data_schema.json" - json_data = json.loads(json.dumps(temp_data)) - with open(schema_file, "r") as f: - json_schema = json.load(f) - try: - # Suppressing writing of data2.json. Can use it for debugging - # with open('data2.json', 'w') as outfile: - # json.dump(temp_data, outfile, sort_keys=True, indent=4) - jsonschema.validate(json_data, json_schema) - except jsonschema.exceptions.ValidationError as e: - LOG.error("Validation Error") - LOG.error("Message:{}".format(e.message)) - LOG.error("Validator_path:{}".format(e.path)) - LOG.error("Validator_pattern:{}".format(e.validator_value)) - LOG.error("Validator:{}".format(e.validator)) - sys.exit() - except jsonschema.exceptions.SchemaError as e: - LOG.error("Schema Validation Error!!") - LOG.error("Message:{}".format(e.message)) - LOG.error("Schema:{}".format(e.schema)) - LOG.error("Validator_value:{}".format(e.validator_value)) - LOG.error("Validator:{}".format(e.validator)) - LOG.error("path:{}".format(e.path)) - sys.exit() + validator = Draft7Validator(self.intermediary_schema) + errors = sorted( + validator.iter_errors(self.data.dict_from_class()), + key=lambda e: e.path) + if errors: + raise exceptions.IntermediaryValidationException(errors=errors) LOG.info("Data validation Passed!") + return def _apply_design_rules(self): """Applies design rules from rules.yaml @@ -309,5 +289,6 @@ class ProcessDataSource(object): self._apply_design_rules() self._get_genesis_node_details() # This will validate the extracted data from different sources. - # self._validate_intermediary_data(self.data) + if not self.no_validation and self.intermediary_schema: + self._validate_intermediary_data() return self.data diff --git a/spyglass/schemas/data_schema.json b/spyglass/schemas/intermediary_schema.json similarity index 99% rename from spyglass/schemas/data_schema.json rename to spyglass/schemas/intermediary_schema.json index 1df9a93..84d13ba 100644 --- a/spyglass/schemas/data_schema.json +++ b/spyglass/schemas/intermediary_schema.json @@ -8,7 +8,7 @@ "type": "object", "properties": { "baremetal": { - "type": "array", + "type": "object", "items": { "type": "object", "$ref": "#/definitions/baremetal_list" diff --git a/tests/conftest.py b/tests/conftest.py index 50cbd01..e60549a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,6 +29,15 @@ def site_document_data_objects(request): request.cls.site_document_data = site_document_data_factory(yaml_data) +@pytest.fixture(scope='class') +def invalid_site_document_data_objects(request): + with open(os.path.join(FIXTURE_DIR, 'invalid_intermediary.yaml'), + 'r') as f: + yaml_data = yaml.safe_load(f) + request.cls.invalid_site_document_data = site_document_data_factory( + yaml_data) + + @pytest.fixture(scope='class') def rules_data(request): with open(os.path.join(FIXTURE_DIR, 'rules.yaml'), 'r') as f: diff --git a/tests/shared/intermediary_schema.json b/tests/shared/intermediary_schema.json new file mode 100644 index 0000000..84d13ba --- /dev/null +++ b/tests/shared/intermediary_schema.json @@ -0,0 +1,373 @@ +{ + "$schema": "http://json-schema.org/schema#", + "metadata": { + "name": "spyglass/Intermediary/v1" + }, + "title": "All", + "description": "All information", + "type": "object", + "properties": { + "baremetal": { + "type": "object", + "items": { + "type": "object", + "$ref": "#/definitions/baremetal_list" + } + }, + "network": { + "type": "object", + "properties": { + "bgp": { + "type": "object", + "$ref": "#/definitions/bgp" + }, + "vlan_network_data": { + "type": "array", + "$ref": "#/definitions/vlan_network_data" + } + }, + "required": [ + "bgp", + "vlan_network_data" + ] + }, + "site_info": { + "type": "object", + "$ref": "#/definitions/site_info" + }, + "storage": { + "type": "object", + "$ref": "#/definitions/storage" + } + }, + "required": [ + "baremetal", + "network", + "site_info", + "storage" + ], + "definitions": { + "baremetal_list": { + "type": "object", + "patternProperties": { + ".*": { + "properties": { + "ip": { + "type": "object", + "properties": { + "calico": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" + }, + "oam": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" + }, + "oob": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" + }, + "overlay": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" + }, + "pxe": { + "type": "string", + "pattern": "^((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(#CHANGE_ME)$" + }, + "storage": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" + } + }, + "required" :[ + "calico", + "oam", + "oob", + "overlay", + "pxe", + "storage" + ] + }, + "host_profile": { + "description": "Host profile of the host", + "type": "string", + "pattern": "^([a-zA-Z]+)|(#CHANGE_ME)$" + }, + "type": { + "description": "Host profile type:Compute or Controller or genesis ", + "type": "string", + "pattern": "^(?i)compute|controller|genesis$" + } + }, + "required" :[ + "ip", + "host_profile", + "type" + ] + } + } + }, + "bgp": { + "type": "object", + "properties": { + "asnumber": { + "type": "integer", + "pattern": "^[0-9]{1,10}$" + }, + "peer_asnumber": { + "type": "integer", + "pattern": "^[0-9]{1,10}$" + }, + "peers": { + "type": "array", + "items": [ + { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$" + } + ] + } + }, + "required": [ + "asnumber", + "peer_asnumber", + "peers" + ] + }, + "vlan_network_data": { + "type": "object", + "properties": { + "calico": { + "type": "object", + "properties": { + "subnet": { + "description": "Subnet address of the network", + "type": "array", + "items": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } + }, + "vlan": { + "description": "Vlan id of the network", + "type": "string", + "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])$" + } + }, + "required": [ + "subnet", + "vlan" + ] + }, + "ingress": { + "type": "object", + "properties": { + "subnet": { + "description": "Subnet address of the network", + "type": "array", + "items": [ + { + "type": "string", + "pattern":"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } + ] + } + }, + "required": [ + "subnet" + ] + }, + "oam": { + "type": "object", + "properties": { + "subnet": { + "description": "Subnet address of the network", + "type": "array", + "items": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } + }, + "vlan": { + "description": "Vlan id of the network", + "type": "string", + "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])$" + } + }, + "required": [ + "subnet", + "vlan" + ] + }, + "oob": { + "type": "object", + "properties": { + "subnet": { + "description": "Subnet address of the network", + "type": "array", + "items": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } + }, + "vlan": { + "description": "Vlan id of the network", + "type": "string", + "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])?$" + } + }, + "required": [ + "subnet" + ] + }, + "pxe": { + "type": "object", + "properties": { + "subnet": { + "description": "Subnet address of the network", + "type": "array", + "items": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } + }, + "vlan": { + "description": "Vlan id of the network", + "type": "string", + "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])$" + } + }, + "required": [ + "subnet", + "vlan" + ] + }, + "storage": { + "type": "object", + "properties": { + "subnet": { + "description": "Subnet address of the network", + "type": "array", + "items": { + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])/([0-9]|[1-2][0-9]|3[0-2])$" + } + }, + "vlan": { + "description": "Vlan id of the network", + "type": "string", + "pattern": "^([0-9]|[0-9][0-9]|[0-9][0-9][0-9]|[0-3][0-9][0-9][0-9]|40[0-9][0-5])$" + } + }, + "required": [ + "subnet", + "vlan" + ] + } + }, + "required" :[ + "calico", + "ingress", + "oam", + "oob", + "overlay", + "pxe", + "storage" + ] + }, + "site_info": { + "type": "object", + "properties": { + "dns": { + "type": "object", + "properties": { + "servers": { + "type": "string", + "pattern": "^((((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]),)+)|(((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))+))+$" + } + } + }, + "ntp": { + "type": "object", + "properties": { + "servers": { + "type": "string", + "pattern": "^((((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]),)+)|(((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))+))+$" + } + } + }, + "ldap": { + "type": "object", + "properties": { + "common_name": { + "type": "string", + "pattern": "\\W+|\\w+" + }, + "subdomain": { + "type": "string", + "pattern": "(?i)\\w+" + }, + "url": { + "type": "string", + "pattern": "^\\w+://\\w+.*\\.[a-zA-Z]{2,3}$" + } + }, + "required": [ + "common_name", + "subdomain", + "url" + ] + }, + "country": { + "type": "string", + "pattern": "(?i)\\w+" + }, + "name": { + "type": "string", + "pattern": "(?i)\\w+" + }, + "state": { + "type": "string", + "pattern": "(?i)\\w+" + }, + "sitetype": { + "type": "string", + "pattern": "(?i)\\w+" + }, + "physical_location_id": { + "type": "string", + "pattern": "^\\w+" + }, + "domain": { + "type": "string", + "pattern": "^\\w+.*\\.[a-zA-Z]{2,3}$" + } + }, + "required": [ + "dns", + "ntp", + "ldap", + "country", + "name", + "state", + "sitetype", + "physical_location_id", + "domain" + ] + }, + "storage": { + "type": "object", + "patternProperties": { + "ceph": { + "controller": { + "osd_count": { + "type": "integer", + "pattern": "^[0-9]{1,2}$" + } + } + } + } + } + } +} diff --git a/tests/shared/invalid_intermediary.yaml b/tests/shared/invalid_intermediary.yaml new file mode 100644 index 0000000..ca0d9e3 --- /dev/null +++ b/tests/shared/invalid_intermediary.yaml @@ -0,0 +1,216 @@ +baremetal: + rack72: + cab2r72c12: + host_profile: dp-r720 + ip: + calico: 30.29.1.12 + oam: 10.0.220.12 + oob: 10.0.220.140 + overlay: 30.19.0.12 + pxe: 30.30.4.12 + storage: 30.31.1.12 + type: compute + cab2r72c13: + host_profile: dp-r720 + ip: + calico: 30.29.1.13 + oam: 10.0.220.13 + oob: 10.0.220.141 + overlay: 30.19.0.13 + pxe: 30.30.4.13 + storage: 30.31.1.13 + type: compute + cab2r72c14: + host_profile: dp-r720 + ip: + calico: 30.29.1.14 + oam: 10.0.220.14 + oob: 10.0.220.142 + overlay: 30.19.0.14 + pxe: 30.30.4.14 + storage: 30.31.1.14 + type: compute + cab2r72c15: + host_profile: dp-r720 + ip: + calico: 30.29.1.15 + oam: 10.0.220.15 + oob: 10.0.220.143 + overlay: 30.19.0.15 + pxe: 30.30.4.15 + storage: 30.31.1.15 + type: compute + cab2r72c16: + host_profile: cp-r720 + ip: + calico: 30.29.1.16 + oam: 10.0.220.16 + oob: 10.0.220.144 + overlay: 30.19.0.16 + pxe: 30.30.4.16 + storage: 30.31.1.16 + name: cab2r72c16 + type: genesis + cab2r72c17: + host_profile: cp-r720 + ip: + calico: 30.29.1.17 + oam: 10.0.220.17 + oob: 10.0.220.145 + overlay: 30.19.0.17 + pxe: 30.30.4.17 + storage: 30.31.1.17 + type: controller + rack73: + cab2r73c12: + host_profile: dp-r720 + ip: + calico: 30.29.1.18 + oam: 10.0.220.18 + oob: 10.0.220.146 + overlay: 30.19.0.18 + pxe: 30.30.4.18 + storage: 30.31.1.18 + type: compute + cab2r73c13: + host_profile: dp-r720 + ip: + calico: 30.29.1.19 + oam: 10.0.220.19 + oob: 10.0.220.147 + overlay: 30.19.0.19 + pxe: 30.30.4.19 + storage: 30.31.1.19 + type: compute + cab2r73c14: + host_profile: dp-r720 + ip: + calico: 30.29.1.20 + oam: 10.0.220.20 + oob: 10.0.220.148 + overlay: 30.19.0.20 + pxe: 30.30.4.20 + storage: 30.31.1.20 + type: compute + cab2r73c15: + host_profile: dp-r720 + ip: + calico: 30.29.1.21 + oam: 10.0.220.21 + oob: 10.0.220.149 + overlay: 30.19.0.21 + pxe: 30.30.4.21 + storage: 30.31.1.21 + type: compute + cab2r73c16: + host_profile: cp-r720 + ip: + calico: 30.29.1.22 + oam: 10.0.220.22 + oob: 10.0.220.150 + overlay: 30.19.0.22 + pxe: 30.30.4.22 + storage: 30.31.1.22 + type: controller + cab2r73c17: + host_profile: cp-r720 + ip: + calico: 30.29.1.23 + oam: 10.0.220.23 + oob: 10.0.220.151 + overlay: 30.19.0.23 + pxe: 30.30.4.23 + storage: 30.31.1.23 + type: controller +network: + bgp: + asnumber: 64671 + ingress_vip: 132.68.226.73 + peer_asnumber: 64688 + peers: + - 172.29.0.2 + - 172.29.0.3 + public_service_cidr: 132.68.226.72/29 + vlan_network_data: + calico: + gateway: 30.29.1.1 + reserved_end: 30.29.1.12 + reserved_start: 30.29.1.1 + routes: [] + static_end: 30.29.1.126 + static_start: 30.29.1.13 + subnet: + - 30.29.1.0/25 + vlan: '22' + ingress: + subnet: + - 132.68.226.72/29 + oob: + gateway: 10.0.220.129 + reserved_end: 10.0.220.138 + reserved_start: 10.0.220.129 + routes: [] + static_end: 10.0.220.158 + static_start: 10.0.220.139 + subnet: + - 10.0.220.128/27 + - 10.0.220.160/27 + - 10.0.220.192/27 + - 10.0.220.224/27 + overlay: + gateway: 30.19.0.1 + reserved_end: 30.19.0.12 + reserved_start: 30.19.0.1 + routes: [] + static_end: 30.19.0.126 + static_start: 30.19.0.13 + subnet: + - 30.19.0.0/25 + vlan: '24' + pxe: + dhcp_end: 30.30.4.126 + dhcp_start: 30.30.4.64 + gateway: 30.30.4.1 + reserved_end: 30.30.4.12 + reserved_start: 30.30.4.1 + routes: [] + static_end: 30.30.4.63 + static_start: 30.30.4.13 + subnet: + - 30.30.4.0/25 + - 30.30.4.128/25 + - 30.30.5.0/25 + - 30.30.5.128/25 + vlan: '21' + storage: + gateway: 30.31.1.1 + reserved_end: 30.31.1.12 + reserved_start: 30.31.1.1 + routes: [] + static_end: 30.31.1.126 + static_start: 30.31.1.13 + subnet: + - 30.31.1.0/25 + vlan: '23' +region_name: test +site_info: + corridor: c1 + country: SampleCountry + dns: + servers: 8.8.8.8,8.8.4.4,208.67.222.222 + domain: atlantafoundry.com + ldap: + common_name: test + domain: example + subdomain: test + url: ldap://ldap.example.com + name: SampleSiteName + ntp: + servers: 10.10.10.10,20.20.20.20,30.30.30.30 + physical_location_id: XXXXXX21 + sitetype: foundry + state: New Jersey +storage: + ceph: + controller: + osd_count: 6 diff --git a/tests/unit/parser/test_engine.py b/tests/unit/parser/test_engine.py index fa3082b..b6e120f 100644 --- a/tests/unit/parser/test_engine.py +++ b/tests/unit/parser/test_engine.py @@ -19,6 +19,7 @@ from unittest import mock from netaddr import IPNetwork from pytest import mark +from spyglass import exceptions from spyglass.parser.engine import ProcessDataSource FIXTURE_DIR = os.path.join( @@ -27,6 +28,7 @@ FIXTURE_DIR = os.path.join( @mark.usefixtures('tmpdir') @mark.usefixtures('site_document_data_objects') +@mark.usefixtures('invalid_site_document_data_objects') @mark.usefixtures('rules_data') class TestProcessDataSource(unittest.TestCase): REGION_NAME = 'test' @@ -69,9 +71,20 @@ class TestProcessDataSource(unittest.TestCase): obj._get_genesis_node_details() self.assertEqual(expected_result, obj.genesis_node) - @unittest.skip('Not in use.') def test__validate_intermediary_data(self): - pass + schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json') + obj = ProcessDataSource( + self.REGION_NAME, self.site_document_data, schema_path, False) + result = obj._validate_intermediary_data() + self.assertIsNone(result) + + def test__validate_intermediary_data_invalid(self): + schema_path = os.path.join(FIXTURE_DIR, 'intermediary_schema.json') + obj = ProcessDataSource( + self.REGION_NAME, self.invalid_site_document_data, schema_path, + False) + with self.assertRaises(exceptions.IntermediaryValidationException): + obj._validate_intermediary_data() @mock.patch.object(ProcessDataSource, '_apply_rule_ip_alloc_offset') @mock.patch.object(ProcessDataSource, '_apply_rule_hardware_profile')