diff --git a/spyglass/cli.py b/spyglass/cli.py index 7ff6eb8..9dacd99 100644 --- a/spyglass/cli.py +++ b/spyglass/cli.py @@ -20,6 +20,7 @@ from click_plugins import with_plugins import pkg_resources import yaml +from spyglass import exceptions from spyglass.parser.engine import ProcessDataSource from spyglass.site_processors.site_processor import SiteProcessor from spyglass.validators.json_validator import JSONSchemaValidator @@ -77,20 +78,14 @@ def main(*, verbose): def intermediary_processor(plugin_type, **kwargs): LOG.info("Generating Intermediary yaml") plugin_type = plugin_type - plugin_class = None - # Discover the plugin and load the plugin class + # Load the plugin class LOG.info("Load the plugin class") - for entry_point in \ - pkg_resources.iter_entry_points('data_extractor_plugins'): - LOG.debug("Entry point '%s' found", entry_point.name) - if entry_point.name == plugin_type: - plugin_class = entry_point.load() - - if plugin_class is None: - LOG.error( - "Unsupported Plugin type. Plugin type:{}".format(plugin_type)) - exit() + try: + plugin_class = pkg_resources.load_entry_point( + "spyglass", "data_extractor_plugins", plugin_type) + except ImportError: + raise exceptions.UnsupportedPlugin() # Extract data from plugin data source LOG.info("Extract data from plugin data source") diff --git a/spyglass/exceptions.py b/spyglass/exceptions.py new file mode 100644 index 0000000..ab0a855 --- /dev/null +++ b/spyglass/exceptions.py @@ -0,0 +1,38 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +LOG = logging.getLogger(__name__) + + +class SpyglassBaseException(Exception): + """Base Spyglass exception""" + + message = 'Base Spyglass exception.' + + def __init__(self, message=None, **kwargs): + self.message = message or self.message + try: + self.message = self.message.format(**kwargs) + except KeyError: + LOG.warning('Missing kwargs') + super().__init__(self.message) + + +class UnsupportedPlugin(SpyglassBaseException): + """Exception that occurs when a plugin is called that does not exist""" + message = ( + '%(plugin_name) was not found in the package %(entry_point) ' + 'entry points.') diff --git a/tests/shared/site_config.yaml b/tests/shared/site_config.yaml new file mode 100644 index 0000000..13979f6 --- /dev/null +++ b/tests/shared/site_config.yaml @@ -0,0 +1,33 @@ +############################################## +# Site Specific Spyglass XLS Plugin Settings # +############################################## +--- +site_info: + ldap: + common_name: test + url: ldap://ldap.example.com + subdomain: test + ntp: + servers: 10.10.10.10,20.20.20.20,30.30.30.30 + sitetype: foundry + domain: atlantafoundry.com + dns: + servers: 8.8.8.8,8.8.4.4,208.67.222.222 +network: + vlan_network_data: + ingress: + subnet: + - 132.68.226.72/29 + bgp : + peers: + - '172.29.0.2' + - '172.29.0.3' + asnumber: 64671 + peer_asnumber: 64688 +storage: + ceph: + controller: + osd_count: 6 +... + + diff --git a/tests/shared/templates/site-definition.yaml.j2 b/tests/shared/templates/site-definition.yaml.j2 new file mode 100644 index 0000000..d446262 --- /dev/null +++ b/tests/shared/templates/site-definition.yaml.j2 @@ -0,0 +1,13 @@ +--- +schema: pegleg/SiteDefinition/v1 +metadata: + schema: metadata/Document/v1 + layeringDefinition: + abstract: false + layer: site + name: {{ data['region_name'] }} + storagePolicy: cleartext +data: + site_type: {{ data['site_info']['sitetype'] }} +... + diff --git a/tests/shared/test_intermediary.yaml b/tests/shared/test_intermediary.yaml new file mode 100644 index 0000000..706c865 --- /dev/null +++ b/tests/shared/test_intermediary.yaml @@ -0,0 +1,227 @@ +baremetal: + rack72: + cab2r72c12: + host_profile: dp-r720 + ip: + calico: 30.29.1.12 + oam: 10.0.220.12 + oob: 10.0.220.140 + overlay: 30.19.0.12 + pxe: 30.30.4.12 + storage: 30.31.1.12 + type: compute + cab2r72c13: + host_profile: dp-r720 + ip: + calico: 30.29.1.13 + oam: 10.0.220.13 + oob: 10.0.220.141 + overlay: 30.19.0.13 + pxe: 30.30.4.13 + storage: 30.31.1.13 + type: compute + cab2r72c14: + host_profile: dp-r720 + ip: + calico: 30.29.1.14 + oam: 10.0.220.14 + oob: 10.0.220.142 + overlay: 30.19.0.14 + pxe: 30.30.4.14 + storage: 30.31.1.14 + type: compute + cab2r72c15: + host_profile: dp-r720 + ip: + calico: 30.29.1.15 + oam: 10.0.220.15 + oob: 10.0.220.143 + overlay: 30.19.0.15 + pxe: 30.30.4.15 + storage: 30.31.1.15 + type: compute + cab2r72c16: + host_profile: cp-r720 + ip: + calico: 30.29.1.16 + oam: 10.0.220.16 + oob: 10.0.220.144 + overlay: 30.19.0.16 + pxe: 30.30.4.16 + storage: 30.31.1.16 + name: cab2r72c16 + type: genesis + cab2r72c17: + host_profile: cp-r720 + ip: + calico: 30.29.1.17 + oam: 10.0.220.17 + oob: 10.0.220.145 + overlay: 30.19.0.17 + pxe: 30.30.4.17 + storage: 30.31.1.17 + type: controller + rack73: + cab2r73c12: + host_profile: dp-r720 + ip: + calico: 30.29.1.18 + oam: 10.0.220.18 + oob: 10.0.220.146 + overlay: 30.19.0.18 + pxe: 30.30.4.18 + storage: 30.31.1.18 + type: compute + cab2r73c13: + host_profile: dp-r720 + ip: + calico: 30.29.1.19 + oam: 10.0.220.19 + oob: 10.0.220.147 + overlay: 30.19.0.19 + pxe: 30.30.4.19 + storage: 30.31.1.19 + type: compute + cab2r73c14: + host_profile: dp-r720 + ip: + calico: 30.29.1.20 + oam: 10.0.220.20 + oob: 10.0.220.148 + overlay: 30.19.0.20 + pxe: 30.30.4.20 + storage: 30.31.1.20 + type: compute + cab2r73c15: + host_profile: dp-r720 + ip: + calico: 30.29.1.21 + oam: 10.0.220.21 + oob: 10.0.220.149 + overlay: 30.19.0.21 + pxe: 30.30.4.21 + storage: 30.31.1.21 + type: compute + cab2r73c16: + host_profile: cp-r720 + ip: + calico: 30.29.1.22 + oam: 10.0.220.22 + oob: 10.0.220.150 + overlay: 30.19.0.22 + pxe: 30.30.4.22 + storage: 30.31.1.22 + type: controller + cab2r73c17: + host_profile: cp-r720 + ip: + calico: 30.29.1.23 + oam: 10.0.220.23 + oob: 10.0.220.151 + overlay: 30.19.0.23 + pxe: 30.30.4.23 + storage: 30.31.1.23 + type: controller +network: + bgp: + asnumber: 64671 + ingress_vip: 132.68.226.73 + peer_asnumber: 64688 + peers: + - 172.29.0.2 + - 172.29.0.3 + public_service_cidr: 132.68.226.72/29 + vlan_network_data: + calico: + gateway: 30.29.1.1 + reserved_end: 30.29.1.12 + reserved_start: 30.29.1.1 + routes: [] + static_end: 30.29.1.126 + static_start: 30.29.1.13 + subnet: + - 30.29.1.0/25 + vlan: '22' + ingress: + subnet: + - 132.68.226.72/29 + oam: + gateway: 10.0.220.1 + reserved_end: 10.0.220.12 + reserved_start: 10.0.220.1 + routes: + - 0.0.0.0/0 + static_end: 10.0.220.62 + static_start: 10.0.220.13 + subnet: + - 10.0.220.0/26 + vlan: '21' + oob: + gateway: 10.0.220.129 + reserved_end: 10.0.220.138 + reserved_start: 10.0.220.129 + routes: [] + static_end: 10.0.220.158 + static_start: 10.0.220.139 + subnet: + - 10.0.220.128/27 + - 10.0.220.160/27 + - 10.0.220.192/27 + - 10.0.220.224/27 + overlay: + gateway: 30.19.0.1 + reserved_end: 30.19.0.12 + reserved_start: 30.19.0.1 + routes: [] + static_end: 30.19.0.126 + static_start: 30.19.0.13 + subnet: + - 30.19.0.0/25 + vlan: '24' + pxe: + dhcp_end: 30.30.4.126 + dhcp_start: 30.30.4.64 + gateway: 30.30.4.1 + reserved_end: 30.30.4.12 + reserved_start: 30.30.4.1 + routes: [] + static_end: 30.30.4.63 + static_start: 30.30.4.13 + subnet: + - 30.30.4.0/25 + - 30.30.4.128/25 + - 30.30.5.0/25 + - 30.30.5.128/25 + vlan: '21' + storage: + gateway: 30.31.1.1 + reserved_end: 30.31.1.12 + reserved_start: 30.31.1.1 + routes: [] + static_end: 30.31.1.126 + static_start: 30.31.1.13 + subnet: + - 30.31.1.0/25 + vlan: '23' +region_name: test +site_info: + corridor: c1 + country: SampleCountry + dns: + servers: 8.8.8.8,8.8.4.4,208.67.222.222 + domain: atlantafoundry.com + ldap: + common_name: test + domain: example + subdomain: test + url: ldap://ldap.example.com + name: SampleSiteName + ntp: + servers: 10.10.10.10,20.20.20.20,30.30.30.30 + physical_location_id: XXXXXX21 + sitetype: foundry + state: New Jersey +storage: + ceph: + controller: + osd_count: 6 diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 0000000..2346bdb --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,143 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from unittest import mock + +from click.testing import CliRunner +import pytest +import yaml + +from spyglass.cli import generate_manifests_using_intermediary +from spyglass.cli import intermediary_processor +from spyglass.cli import validate_manifests_against_schemas +from spyglass import exceptions +from spyglass.parser.engine import ProcessDataSource +from spyglass.site_processors.site_processor import SiteProcessor +from spyglass.validators.json_validator import JSONSchemaValidator + +FIXTURE_DIR = os.path.join( + os.path.dirname(os.path.dirname(__file__)), 'shared') + +INTERMEDIARY_PATH = os.path.join(FIXTURE_DIR, 'test_intermediary.yaml') + +TEMPLATE_DIR_PATH = os.path.join(FIXTURE_DIR, 'templates') + +SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml') + +DOCUMENTS_PATH = os.path.join(FIXTURE_DIR, 'documents') + +SCHEMAS_PATH = os.path.join(FIXTURE_DIR, 'schemas') + + +def _get_intermediary_process_kwargs(): + return {'site_name': 'test'} + + +def _get_site_config_data(): + with open(SITE_CONFIG_PATH, 'r') as f: + data = f.read() + return yaml.safe_load(data) + + +def _get_intermediary_data(): + with open(INTERMEDIARY_PATH, 'r') as f: + data = f.read() + return yaml.safe_load(data) + + +@mock.patch('spyglass.parser.engine.ProcessDataSource', autospec=True) +@mock.patch('spyglass_plugin_xls.excel.ExcelPlugin', autospec=True) +def test_intermediary_processor(mock_excel_plugin, mock_process_data_source): + """Tests that the intermediary processor produces expected results""" + plugin_name = 'excel' + data = _get_intermediary_process_kwargs() + mock_excel_plugin.return_value.site_data = {} + result = intermediary_processor(plugin_name, **data) + assert type(result) == ProcessDataSource + + +def test_intermediary_processor_unsupported_plugin(): + """Tests that an exception is raised if a plugin is not configured""" + plugin_name = 'invalid_plugin' + with pytest.raises(exceptions.UnsupportedPlugin): + intermediary_processor( + plugin_name, **_get_intermediary_process_kwargs()) + + +@mock.patch('spyglass.parser.engine.ProcessDataSource', autospec=True) +@mock.patch('spyglass_plugin_xls.excel.ExcelPlugin', autospec=False) +def test_intermediary_processor_additional_config( + mock_excel_plugin, mock_process_data_source): + """Tests that an additional configuration is processed if included""" + plugin_name = 'excel' + data = _get_intermediary_process_kwargs() + data['site_configuration'] = SITE_CONFIG_PATH + mock_excel_plugin.return_value.site_data = {} + result = intermediary_processor(plugin_name, **data) + assert type(result) == ProcessDataSource + mock_excel_plugin.return_value.apply_additional_data.\ + assert_called_once_with(_get_site_config_data()) + + +@mock.patch.object( + SiteProcessor, '__init__', spec=SiteProcessor, return_value=None) +def test_generate_manifests_using_intermediary(mock_site_processor): + """Tests `mi` command from CLI""" + runner = CliRunner() + with mock.patch.object(SiteProcessor, 'render_template', + spec=SiteProcessor) as mock_render: + result = runner.invoke( + generate_manifests_using_intermediary, + [INTERMEDIARY_PATH, '-t', TEMPLATE_DIR_PATH]) + assert result.exit_code == 0 + mock_site_processor.assert_called_once_with( + _get_intermediary_data(), None, False) + mock_render.assert_called_once_with(TEMPLATE_DIR_PATH) + + +def test_generate_manifests_using_intermediary_no_intermediary_file(): + """Tests bad input for intermediary file for `mi` command""" + runner = CliRunner() + with mock.patch.object(SiteProcessor, 'render_template', + spec=SiteProcessor) as mock_render: + result = runner.invoke( + generate_manifests_using_intermediary, ['-t', TEMPLATE_DIR_PATH]) + assert result.exit_code != 0 + assert not mock_render.called + + +def test_generate_manifests_using_intermediary_no_templates(): + """Tests bad input for templates folder for `mi` command""" + runner = CliRunner() + with mock.patch.object(SiteProcessor, 'render_template', + spec=SiteProcessor) as mock_render: + result = runner.invoke( + generate_manifests_using_intermediary, [INTERMEDIARY_PATH]) + assert result.exit_code != 0 + assert not mock_render.called + + +@mock.patch.object( + JSONSchemaValidator, '__init__', autospec=True, return_value=None) +@mock.patch.object(JSONSchemaValidator, 'validate', autospec=True) +def test_validate_manifests_against_schemas(mock_validate, mock_validator): + """Tests `validate` command from CLI using defualt behavior""" + runner = CliRunner() + result = runner.invoke( + validate_manifests_against_schemas, + ['-d', DOCUMENTS_PATH, '-p', SCHEMAS_PATH]) + assert result.exit_code == 0 + mock_validator.assert_called_once_with( + mock.ANY, DOCUMENTS_PATH, SCHEMAS_PATH) + mock_validate.assert_called_once() diff --git a/tox.ini b/tox.ini index 3fabcef..54ed7be 100644 --- a/tox.ini +++ b/tox.ini @@ -72,6 +72,6 @@ deps = commands = bash -c 'PATH=$PATH:~/.local/bin; pytest --cov=spyglass --cov-report \ html:cover --cov-report xml:cover/coverage.xml --cov-report term \ - --cov-fail-under 10 tests/' + --cov-fail-under 50 tests/' whitelist_externals = bash