From 60da55cd18c01a6bd00f3ad6852c9d28054dd3e3 Mon Sep 17 00:00:00 2001 From: "Ian H. Pittwood" Date: Tue, 14 May 2019 16:26:30 -0500 Subject: [PATCH] Schema validation Adds JSON schema validation to Spyglass. Change-Id: Ib29bbf9fa02cd6623c75db37a4c8d6f510b52831 --- spyglass/cli.py | 26 ++ spyglass/schemas/data_schema.json | 3 + spyglass/validators/__init__.py | 0 spyglass/validators/exceptions.py | 32 ++ spyglass/validators/json_validator.py | 164 +++++++++ spyglass/validators/validator.py | 33 ++ test-requirements.txt | 5 + tests/__init__.py | 0 tests/shared/documents/invalid/invalid.yaml | 10 + .../valid/PKICatalogue/pki-catalogue.yaml | 342 ++++++++++++++++++ .../valid/SiteDefinition/site-definition.yaml | 12 + .../schemas/InvalidSchema/invalid-schema.yaml | 20 + .../PKICatalogue/pki-catalogue-schema.yaml | 42 +++ .../site-definition-schema.yaml | 27 ++ tests/unit/__init__.py | 0 tests/unit/validators/__init__.py | 0 tests/unit/validators/test_json_validator.py | 89 +++++ tools/gate/run-unit-tests.sh | 12 + tox.ini | 21 +- 19 files changed, 834 insertions(+), 4 deletions(-) create mode 100644 spyglass/validators/__init__.py create mode 100644 spyglass/validators/exceptions.py create mode 100644 spyglass/validators/json_validator.py create mode 100644 spyglass/validators/validator.py create mode 100644 tests/__init__.py create mode 100644 tests/shared/documents/invalid/invalid.yaml create mode 100644 tests/shared/documents/valid/PKICatalogue/pki-catalogue.yaml create mode 100644 tests/shared/documents/valid/SiteDefinition/site-definition.yaml create mode 100644 tests/shared/schemas/InvalidSchema/invalid-schema.yaml create mode 100644 tests/shared/schemas/PKICatalogue/pki-catalogue-schema.yaml create mode 100644 tests/shared/schemas/SiteDefinition/site-definition-schema.yaml create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/validators/__init__.py create mode 100644 tests/unit/validators/test_json_validator.py create mode 100755 tools/gate/run-unit-tests.sh diff --git a/spyglass/cli.py b/spyglass/cli.py index 81369d8..b98f057 100644 --- a/spyglass/cli.py +++ b/spyglass/cli.py @@ -22,6 +22,7 @@ import yaml from spyglass.parser.engine import ProcessDataSource from spyglass.site_processors.site_processor import SiteProcessor +from spyglass.validators.json_validator import JSONSchemaValidator LOG = logging.getLogger(__name__) @@ -133,3 +134,28 @@ def generate_manifests_using_intermediary( LOG.info("Generating site Manifests") processor_engine = SiteProcessor(intermediary_yaml, manifest_dir) processor_engine.render_template(template_dir) + + +@main.command( + 'validate', + short_help='validates pegleg documents', + help='Validates pegleg documents against their schema.') +@click.option( + '-d', + '--document-path', + 'document_path', + type=click.Path(exists=True, readable=True), + required=True, + help='Path to the documents to validate.') +@click.option( + '-p', + '--schema-path', + 'schema_path', + type=click.Path(exists=True, readable=True), + required=True, + help=( + 'Path to a schema file or directory of schema files used to ' + 'validate documents.')) +def validate_manifests_against_schemas(document_path, schema_path): + validator = JSONSchemaValidator(document_path, schema_path) + validator.validate() diff --git a/spyglass/schemas/data_schema.json b/spyglass/schemas/data_schema.json index 7be761f..1df9a93 100644 --- a/spyglass/schemas/data_schema.json +++ b/spyglass/schemas/data_schema.json @@ -1,5 +1,8 @@ { "$schema": "http://json-schema.org/schema#", + "metadata": { + "name": "spyglass/Intermediary/v1" + }, "title": "All", "description": "All information", "type": "object", diff --git a/spyglass/validators/__init__.py b/spyglass/validators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spyglass/validators/exceptions.py b/spyglass/validators/exceptions.py new file mode 100644 index 0000000..9a69293 --- /dev/null +++ b/spyglass/validators/exceptions.py @@ -0,0 +1,32 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class PathDoesNotExistError(OSError): + """Exception that occurs when the document or schema path does not exist""" + pass + + +class UnexpectedFileType(OSError): + """Exception that occurs when an unexpected file type is given""" + pass + + +class DirectoryEmptyError(OSError): + """Exception for when a directory is empty + + This exception can occur when either a directory is empty or if a directory + does not have any files with the correct file extension. + """ + pass diff --git a/spyglass/validators/json_validator.py b/spyglass/validators/json_validator.py new file mode 100644 index 0000000..592221b --- /dev/null +++ b/spyglass/validators/json_validator.py @@ -0,0 +1,164 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from glob import glob +import logging +import os + +from jsonschema import Draft7Validator +import yaml + +from spyglass.validators import exceptions +from spyglass.validators.validator import BaseDocumentValidator + +LOG = logging.getLogger(__name__) + +LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:' \ + '%(funcName)s [%(lineno)3d] %(message)s' + + +class JSONSchemaValidator(BaseDocumentValidator): + """Validator for validating documents using jsonschema""" + + def __init__( + self, + document_path, + schema_path, + document_extension='.yaml', + schema_extension='.yaml', + document_loader=yaml.safe_load, + schema_loader=yaml.safe_load): + super().__init__() + + # Check that given paths are valid + if not os.path.exists(document_path): + LOG.error('Document path: %s does not exist.', document_path) + raise exceptions.PathDoesNotExistError() + if not os.path.exists(schema_path): + LOG.error('Schema path: %s does not exist.', document_path) + raise exceptions.PathDoesNotExistError() + + # Extract list of document file paths from path + if os.path.isdir(document_path): + # Create match string and use glob to generate list of file paths + match = os.path.join(document_path, '**', '*' + document_extension) + self.documents = glob(match, recursive=True) + + # Directory should not be empty + if not self.documents: + LOG.error( + 'No files with %s extension found in document path ' + '%s', document_extension, document_path) + raise exceptions.DirectoryEmptyError() + elif os.path.splitext(document_path) == document_extension: + # Single files can just be appended to the list to process the same + # so long as the extension matches + self.documents.append(document_path) + else: + # Throw error if unexpected file type given + raise exceptions.UnexpectedFileType() + + # Extract list of schema file paths from path + if os.path.isdir(schema_path): + # Create match string and use glob to generate list of file paths + match = os.path.join(schema_path, '**', '*' + schema_extension) + self.schemas = glob(match, recursive=True) + + # Directory should not be empty + if not self.schemas: + LOG.error( + 'No files with %s extension found in document path ' + '%s', document_extension, document_path) + raise exceptions.DirectoryEmptyError() + elif os.path.splitext(schema_path) == schema_extension: + # Single files can just be appended to the list to process the same + self.schemas.append(schema_path) + else: + # Throw error if unexpected file type given + raise exceptions.UnexpectedFileType() + + # Initialize pairs list for next step + self.document_schema_pairs = [] + + self.document_loader = document_loader + self.schema_loader = schema_loader + self._match_documents_to_schemas() + + def _match_documents_to_schemas(self): + """Pairs documents to their schemas for easier processing + + Loops through all documents and finds its associated schema using the + "schema" key from documents and the "metadata:name" key from schemas. + Matching document/schema pairs are added to document_schema_pairs. Any + unmatched documents will display a warning. + """ + if not self.documents: + LOG.warning('No documents found.') + + if not self.schemas: + LOG.warning('No schemas found.') + + for document in self.documents: + pair_found = False + with open(document, 'r') as f_doc: + loaded_doc = self.document_loader(f_doc) + if 'schema' in loaded_doc: + schema_name = loaded_doc['schema'] + for schema in self.schemas: + with open(schema, 'r') as f_schema: + loaded_schema = self.schema_loader(f_schema) + if schema_name == loaded_schema['metadata']['name']: + self.document_schema_pairs.append((document, schema)) + pair_found = True + break + else: + LOG.warning('No schema entry found for file %s', document) + if not pair_found: + LOG.warning( + 'No matching schema found for file %s, ' + 'data will not be validated.', document) + + def _validate_file(self, document, schema): + """Validate a document against a schema using JSON Schema Draft 7 + + :param document: File path to the document to validate + :param schema: File path to the schema used to validate document + :return: A list of errors from the validator + """ + with open(document, 'r') as f_doc: + loaded_doc = self.document_loader(f_doc) + with open(schema, 'r') as f_schema: + loaded_schema = self.schema_loader(f_schema) + validator = Draft7Validator(loaded_schema) + return sorted(validator.iter_errors(loaded_doc), key=lambda e: e.path) + + def validate(self): + """Validates document against its schema + + Loops through document_schema_pairs list and validates each pair. Any + errors are logged and returned in a dictionary by file. + + :return: A dictionary of filenames and their list of validation errors + """ + error_list = {} + for document, schema in self.document_schema_pairs: + LOG.info( + 'Validating document %s using schema %s', document, schema) + errors = self._validate_file(document, schema) + if errors: + for error in errors: + LOG.error(error.message) + error_list[document] = errors + + return error_list diff --git a/spyglass/validators/validator.py b/spyglass/validators/validator.py new file mode 100644 index 0000000..9c011ca --- /dev/null +++ b/spyglass/validators/validator.py @@ -0,0 +1,33 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc + + +class BaseDocumentValidator(metaclass=abc.ABCMeta): + """Abstract class for document validation""" + + def __init__(self): + self.documents = [] + self.schemas = [] + + @abc.abstractmethod + def validate(self): + """Validate documents against schemas. + + Runs a validation method on documents, comparing them to schemas for + valid data structure and types. + """ + + return diff --git a/test-requirements.txt b/test-requirements.txt index 81bb406..b79a2b9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,8 @@ +# Testing +pytest==4.4.1 +pytest-xdist==1.28.0 +pytest-cov==2.6.1 + # Formatting yapf==0.27.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/shared/documents/invalid/invalid.yaml b/tests/shared/documents/invalid/invalid.yaml new file mode 100644 index 0000000..8ba3673 --- /dev/null +++ b/tests/shared/documents/invalid/invalid.yaml @@ -0,0 +1,10 @@ +--- +schema: InvalidSchema +foo: Not a number +bar: "Doesn't equal constant" +baz: + staticProperty: + - This array needs at least one number + property1: The propertyNames keyword is an alternative to patternProperties + pr()perty2: "All property names must match supplied conditions (in this" + "case, it's a regex)" diff --git a/tests/shared/documents/valid/PKICatalogue/pki-catalogue.yaml b/tests/shared/documents/valid/PKICatalogue/pki-catalogue.yaml new file mode 100644 index 0000000..a074aed --- /dev/null +++ b/tests/shared/documents/valid/PKICatalogue/pki-catalogue.yaml @@ -0,0 +1,342 @@ +--- +schema: promenade/PKICatalog/v1 +metadata: + schema: metadata/Document/v1 + name: cluster-certificates + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: + certificate_authorities: + kubernetes: + description: CA for Kubernetes components + certificates: + - document_name: apiserver + description: Service certificate for Kubernetes apiserver + common_name: apiserver + hosts: + - localhost + - 127.0.0.1 + - 10.96.0.1 + kubernetes_service_names: + - kubernetes.default.svc.cluster.local + + - document_name: kubelet-genesis + common_name: system:node:cab2r72c16 + hosts: + - cab2r72c16 + - 10.0.220.16 + - + groups: + - system:nodes + - document_name: kubelet-cab2r72c12 + common_name: system:node:cab2r72c12 + hosts: + - cab2r72c12 + - 10.0.220.12 + - + groups: + - system:nodes + - document_name: kubelet-cab2r72c13 + common_name: system:node:cab2r72c13 + hosts: + - cab2r72c13 + - 10.0.220.13 + - + groups: + - system:nodes + - document_name: kubelet-cab2r72c14 + common_name: system:node:cab2r72c14 + hosts: + - cab2r72c14 + - 10.0.220.14 + - + groups: + - system:nodes + - document_name: kubelet-cab2r72c15 + common_name: system:node:cab2r72c15 + hosts: + - cab2r72c15 + - 10.0.220.15 + - + groups: + - system:nodes + - document_name: kubelet-cab2r72c16 + common_name: system:node:cab2r72c16 + hosts: + - cab2r72c16 + - 10.0.220.16 + - + groups: + - system:nodes + - document_name: kubelet-cab2r72c17 + common_name: system:node:cab2r72c17 + hosts: + - cab2r72c17 + - 10.0.220.17 + - + groups: + - system:nodes + - document_name: kubelet-cab2r73c12 + common_name: system:node:cab2r73c12 + hosts: + - cab2r73c12 + - 10.0.220.18 + - + groups: + - system:nodes + - document_name: kubelet-cab2r73c13 + common_name: system:node:cab2r73c13 + hosts: + - cab2r73c13 + - 10.0.220.19 + - + groups: + - system:nodes + - document_name: kubelet-cab2r73c14 + common_name: system:node:cab2r73c14 + hosts: + - cab2r73c14 + - 10.0.220.20 + - + groups: + - system:nodes + - document_name: kubelet-cab2r73c15 + common_name: system:node:cab2r73c15 + hosts: + - cab2r73c15 + - 10.0.220.21 + - + groups: + - system:nodes + - document_name: kubelet-cab2r73c16 + common_name: system:node:cab2r73c16 + hosts: + - cab2r73c16 + - 10.0.220.22 + - + groups: + - system:nodes + - document_name: kubelet-cab2r73c17 + common_name: system:node:cab2r73c17 + hosts: + - cab2r73c17 + - 10.0.220.23 + - + groups: + - system:nodes + - document_name: scheduler + description: Service certificate for Kubernetes scheduler + common_name: system:kube-scheduler + - document_name: controller-manager + description: certificate for controller-manager + common_name: system:kube-controller-manager + - document_name: admin + common_name: admin + groups: + - system:masters + - document_name: armada + common_name: armada + groups: + - system:masters + kubernetes-etcd: + description: Certificates for Kubernetes's etcd servers + certificates: + - document_name: apiserver-etcd + description: etcd client certificate for use by Kubernetes apiserver + common_name: apiserver + - document_name: kubernetes-etcd-anchor + description: anchor + common_name: anchor + - document_name: kubernetes-etcd-genesis + common_name: kubernetes-etcd-genesis + hosts: + - cab2r72c16 + - 10.0.220.16 + - + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - 10.96.0.2 + - document_name: kubernetes-etcd-cab2r72c16 + common_name: kubernetes-etcd-cab2r72c16 + hosts: + - cab2r72c16 + - 10.0.220.16 + - + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - 10.96.0.2 + - document_name: kubernetes-etcd-cab2r72c17 + common_name: kubernetes-etcd-cab2r72c17 + hosts: + - cab2r72c17 + - 10.0.220.17 + - + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - 10.96.0.2 + - document_name: kubernetes-etcd-cab2r73c16 + common_name: kubernetes-etcd-cab2r73c16 + hosts: + - cab2r73c16 + - 10.0.220.22 + - + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - 10.96.0.2 + - document_name: kubernetes-etcd-cab2r73c17 + common_name: kubernetes-etcd-cab2r73c17 + hosts: + - cab2r73c17 + - 10.0.220.23 + - + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - 10.96.0.2 + kubernetes-etcd-peer: + certificates: + - document_name: kubernetes-etcd-genesis-peer + common_name: kubernetes-etcd-genesis-peer + hosts: + - cab2r72c16 + - 10.0.220.16 + - + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - 10.96.0.2 + - document_name: kubernetes-etcd-cab2r72c16-peer + common_name: kubernetes-etcd-cab2r72c16-peer + hosts: + - cab2r72c16 + - 10.0.220.16 + - + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - 10.96.0.2 + - document_name: kubernetes-etcd-cab2r72c17-peer + common_name: kubernetes-etcd-cab2r72c17-peer + hosts: + - cab2r72c17 + - 10.0.220.17 + - + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - 10.96.0.2 + - document_name: kubernetes-etcd-cab2r73c16-peer + common_name: kubernetes-etcd-cab2r73c16-peer + hosts: + - cab2r73c16 + - 10.0.220.22 + - + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - 10.96.0.2 + - document_name: kubernetes-etcd-cab2r73c17-peer + common_name: kubernetes-etcd-cab2r73c17-peer + hosts: + - cab2r73c17 + - 10.0.220.23 + - + - 127.0.0.1 + - localhost + - kubernetes-etcd.kube-system.svc.cluster.local + - 10.96.0.2 + ksn-etcd: + description: Certificates for Calico etcd client traffic + certificates: + - document_name: ksn-etcd-anchor + description: anchor + common_name: anchor + - document_name: ksn-etcd-cab2r72c16 + common_name: ksn-etcd-cab2r72c16 + hosts: + - cab2r72c16 + - 10.0.220.16 + - + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: ksn-etcd-cab2r72c17 + common_name: ksn-etcd-cab2r72c17 + hosts: + - cab2r72c17 + - 10.0.220.17 + - + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: ksn-etcd-cab2r73c16 + common_name: ksn-etcd-cab2r73c16 + hosts: + - cab2r73c16 + - 10.0.220.22 + - + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: ksn-etcd-cab2r73c17 + common_name: ksn-etcd-cab2r73c17 + hosts: + - cab2r73c17 + - 10.0.220.23 + - + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: ksn-node + common_name: calcico-node + ksn-etcd-peer: + description: Certificates for Calico etcd clients + certificates: + - document_name: ksn-etcd-cab2r72c16-peer + common_name: ksn-etcd-cab2r72c16-peer + hosts: + - cab2r72c16 + - 10.0.220.16 + - + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: ksn-etcd-cab2r72c17-peer + common_name: ksn-etcd-cab2r72c17-peer + hosts: + - cab2r72c17 + - 10.0.220.17 + - + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: ksn-etcd-cab2r73c16-peer + common_name: ksn-etcd-cab2r73c16-peer + hosts: + - cab2r73c16 + - 10.0.220.22 + - + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: ksn-etcd-cab2r73c17-peer + common_name: ksn-etcd-cab2r73c17-peer + hosts: + - cab2r73c17 + - 10.0.220.23 + - + - 127.0.0.1 + - localhost + - 10.96.232.136 + - document_name: ksn-node-peer + common_name: calico-node-peer + keypairs: + - name: service-account + description: Service account signing key for use by Kubernetes controller-manager. +... \ No newline at end of file diff --git a/tests/shared/documents/valid/SiteDefinition/site-definition.yaml b/tests/shared/documents/valid/SiteDefinition/site-definition.yaml new file mode 100644 index 0000000..34cbd68 --- /dev/null +++ b/tests/shared/documents/valid/SiteDefinition/site-definition.yaml @@ -0,0 +1,12 @@ +--- +schema: pegleg/SiteDefinition/v1 +metadata: + schema: metadata/Document/v1 + layeringDefinition: + abstract: false + layer: site + name: airship-seaworthy + storagePolicy: cleartext +data: + site_type: foundry +... diff --git a/tests/shared/schemas/InvalidSchema/invalid-schema.yaml b/tests/shared/schemas/InvalidSchema/invalid-schema.yaml new file mode 100644 index 0000000..bd1eac7 --- /dev/null +++ b/tests/shared/schemas/InvalidSchema/invalid-schema.yaml @@ -0,0 +1,20 @@ +--- +metadata: + name: InvalidSchema +type: object +properties: + foo: + type: number + bar: + const: Must equal this value + baz: + type: object + properties: + staticProperty: + type: array + contains: + type: number + propertyNames: + pattern: "^([0-9a-zA-Z]*)$" + additionalProperties: + type: string diff --git a/tests/shared/schemas/PKICatalogue/pki-catalogue-schema.yaml b/tests/shared/schemas/PKICatalogue/pki-catalogue-schema.yaml new file mode 100644 index 0000000..04e9e86 --- /dev/null +++ b/tests/shared/schemas/PKICatalogue/pki-catalogue-schema.yaml @@ -0,0 +1,42 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: promenade/PKICatalog/v1 + labels: + application: promenade +data: + certificate_authorities: + type: array + items: + type: object + properties: + description: + type: string + certificates: + type: array + items: + type: object + properties: + document_name: + type: string + description: + type: string + common_name: + type: string + hosts: + type: array + items: string + groups: + type: array + items: string + keypairs: + type: array + items: + type: object + properties: + name: + type: string + description: + type: string +... \ No newline at end of file diff --git a/tests/shared/schemas/SiteDefinition/site-definition-schema.yaml b/tests/shared/schemas/SiteDefinition/site-definition-schema.yaml new file mode 100644 index 0000000..6023b16 --- /dev/null +++ b/tests/shared/schemas/SiteDefinition/site-definition-schema.yaml @@ -0,0 +1,27 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + schema: metadata/Control/v1 + name: pegleg/SiteDefinition/v1 +data: + type: object + properties: + repositories: + type: object + additionalProperties: + type: object + properties: + revision: + type: string + url: + type: string + required: + - revision + - url + + site_type: + type: string + required: + - site_type + additionalProperties: false +... \ No newline at end of file diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/validators/__init__.py b/tests/unit/validators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/validators/test_json_validator.py b/tests/unit/validators/test_json_validator.py new file mode 100644 index 0000000..e923e34 --- /dev/null +++ b/tests/unit/validators/test_json_validator.py @@ -0,0 +1,89 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from spyglass.validators.exceptions import PathDoesNotExistError +from spyglass.validators.json_validator import JSONSchemaValidator + +FIXTURE_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'shared') + +DOCUMENT_DIR = os.path.join(FIXTURE_DIR, 'documents') + +VALID_DOCUMENTS_DIR = os.path.join(DOCUMENT_DIR, 'valid') + +INVALID_DOCUMENTS_DIR = os.path.join(DOCUMENT_DIR, 'invalid') + +SCHEMA_DIR = os.path.join(FIXTURE_DIR, 'schemas') + + +def test_bad_document_path(): + """Tests that an invalid document path raises a PathDoesNotExistError""" + bad_path = os.path.join(FIXTURE_DIR, 'not_documents') + with pytest.raises(PathDoesNotExistError): + JSONSchemaValidator(bad_path, SCHEMA_DIR) + + +def test_bad_schema_path(): + """Tests that an invalid schema path raises a PathDoesNotExistError""" + bad_path = os.path.join(FIXTURE_DIR, 'not_schemas') + with pytest.raises(PathDoesNotExistError): + JSONSchemaValidator(DOCUMENT_DIR, bad_path) + + +def test_document_schema_matching(): + """Tests that documents and schema are correctly paired up""" + expected_pairs = [ + ('site-definition.yaml', 'site-definition-schema.yaml'), + ('pki-catalogue.yaml', 'pki-catalogue-schema.yaml') + ] + validator = JSONSchemaValidator(VALID_DOCUMENTS_DIR, SCHEMA_DIR) + no_path_pairs = [] + for pair in validator.document_schema_pairs: + no_path_pairs.append( + (os.path.split(pair[0])[1], os.path.split(pair[1])[1])) + assert no_path_pairs == expected_pairs + + +def test_document_schema_matching_no_files(): + """Tests that document and schema are not paired if there are no matches""" + site_definition_doc_dir = os.path.join( + VALID_DOCUMENTS_DIR, 'SiteDefinition') + site_definition_schema_dir = os.path.join(SCHEMA_DIR, 'PKICatalogue') + + expected_pairs = [] + validator = JSONSchemaValidator( + site_definition_doc_dir, site_definition_schema_dir) + no_path_pairs = [] + for pair in validator.document_schema_pairs: + no_path_pairs.append( + (os.path.split(pair[0])[1], os.path.split(pair[1])[1])) + assert no_path_pairs == expected_pairs + + +def test_validate(): + """Tests that validation of correct files yields no errors""" + validator = JSONSchemaValidator(VALID_DOCUMENTS_DIR, SCHEMA_DIR) + errors = validator.validate() + assert not errors + + +def test_validate_with_errors(): + """Tests that correct errors are generated for an invalid document""" + validator = JSONSchemaValidator(INVALID_DOCUMENTS_DIR, SCHEMA_DIR) + errors = validator.validate() + assert errors diff --git a/tools/gate/run-unit-tests.sh b/tools/gate/run-unit-tests.sh new file mode 100755 index 0000000..61fbf44 --- /dev/null +++ b/tools/gate/run-unit-tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e +posargs=$@ +# cross-platform way to derive the number of logical cores +readonly num_cores=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())') +if [ ${#posargs} -ge 1 ]; then + pytest -k ${posargs} -n $num_cores +else + pytest -n $num_cores +fi +set +e diff --git a/tox.ini b/tox.ini index cc1f8a5..3fabcef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = pep8, docs +envlist = py36, py37, pep8, docs, cover minversion = 2.3.1 skipsdist = True @@ -17,13 +17,14 @@ whitelist_externals = find commands = find . -type f -name "*.pyc" -delete + {toxinidir}/tools/gate/run-unit-tests.sh '{posargs}' [testenv:fmt] basepython = python3 deps = -r{toxinidir}/test-requirements.txt commands = - yapf -ir {toxinidir}/spyglass + yapf -ir {toxinidir}/spyglass {toxinidir}/tests [testenv:pep8] basepython = python3 @@ -31,8 +32,8 @@ deps = -r{toxinidir}/test-requirements.txt commands = bash -c {toxinidir}/tools/gate/whitespace-linter.sh - yapf -dr {toxinidir}/spyglass {toxinidir}/setup.py - flake8 {toxinidir}/spyglass + yapf -dr {toxinidir}/spyglass {toxinidir}/setup.py {toxinidir}/tests + flake8 {toxinidir}/spyglass {toxinidir}/tests bandit -r spyglass -n 5 safety check -r requirements.txt --bare whitelist_externals = @@ -62,3 +63,15 @@ commands = rm -rf doc/build sphinx-build -b html doc/source doc/build/html -n -W -v whitelist_externals = rm + +[testenv:cover] +basepython = python3 +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + bash -c 'PATH=$PATH:~/.local/bin; pytest --cov=spyglass --cov-report \ + html:cover --cov-report xml:cover/coverage.xml --cov-report term \ + --cov-fail-under 10 tests/' +whitelist_externals = + bash