diff --git a/requirements.txt b/requirements.txt index 61063be..dd596be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ jsonschema netaddr openpyxl==2.5.4 pyyaml==3.12 -requests \ No newline at end of file +requests +click \ No newline at end of file diff --git a/setup.py b/setup.py index aaa0fb7..2537c43 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( ], entry_points={ 'console_scripts': [ - 'spyglass=spyglass.spyglass:main', + 'spyglass=spyglass.cli:main', ], 'data_extractor_plugins': [ 'formation=' diff --git a/spyglass/cli.py b/spyglass/cli.py new file mode 100644 index 0000000..e5ada66 --- /dev/null +++ b/spyglass/cli.py @@ -0,0 +1,283 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import pprint + +import click +import pkg_resources +import yaml + +from spyglass.parser.engine import ProcessDataSource +from spyglass.site_processors.site_processor import SiteProcessor + +LOG = logging.getLogger(__name__) + +LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:' \ + '%(funcName)s [%(lineno)3d] %(message)s' + +CONTEXT_SETTINGS = { + 'help_option_names': ['-h', '--help'], +} + + +def tugboat_required_callback(ctx, param, value): + LOG.debug('Evaluating %s: %s', param.name, value) + if 'plugin_type' not in ctx.params or \ + ctx.params['plugin_type'] == 'tugboat': + if not value: + raise click.UsageError('%s is required for the tugboat ' + 'plugin.' % str(param.name), + ctx=ctx) + return value + + +def formation_required_callback(ctx, param, value): + LOG.debug('Evaluating %s: %s', param.name, value) + if 'plugin_type' in ctx.params: + if ctx.params['plugin_type'] == 'formation': + if not value: + raise click.UsageError('%s is required for the ' + 'formation plugin.' % str(param.name), + ctx=ctx) + return value + return ['', '', ''] + + +PLUGIN_TYPE_OPTION = click.option( + '-p', + '--plugin-type', + 'plugin_type', + type=click.Choice(['formation', 'tugboat']), + default='tugboat', + show_default=True, + help='The plugin type to use.') # yapf: disable + +# TODO(ianp): Either provide a prompt for passwords or use environment +# variable so passwords are no longer plain text +FORMATION_TARGET_OPTION = click.option( + '-f', + '--formation-target', + 'formation_target', + nargs=3, + help=('Target URL, username, and password for formation plugin. Required ' + 'for formation plugin.'), + callback=formation_required_callback) + +INTERMEDIARY_DIR_OPTION = click.option( + '-d', + '--intermediary-dir', + 'intermediary_dir', + type=click.Path(exists=True, file_okay=False, writable=True), + default='./', + help='Directory in which the intermediary file will be created.') + +EXCEL_FILE_OPTION = click.option( + '-x', + '--excel-file', + 'excel_file', + multiple=True, + type=click.Path(exists=True, readable=True, dir_okay=False), + help='Path to the engineering Excel file. Required for tugboat plugin.', + callback=tugboat_required_callback) + +EXCEL_SPEC_OPTION = click.option( + '-e', + '--excel-spec', + 'excel_spec', + type=click.Path(exists=True, readable=True, dir_okay=False), + help=('Path to the Excel specification YAML file for the engineering ' + 'Excel file. Required for tugboat plugin.'), + callback=tugboat_required_callback) + +SITE_CONFIGURATION_FILE_OPTION = click.option( + '-c', + '--site-configuration', + 'site_configuration', + type=click.Path(exists=True, readable=True, dir_okay=False), + required=False, + help='Path to site specific configuration details YAML file.') + +SITE_NAME_CONFIGURATION_OPTION = click.option( + '-s', + '--site-name', + 'site_name', + type=click.STRING, + required=False, + help='Name of the site for which the intermediary is being generated.') + +TEMPLATE_DIR_OPTION = click.option( + '-t', + '--template-dir', + 'template_dir', + type=click.Path(exists=True, readable=True, file_okay=False), + required=True, + help='Path to the directory containing manifest J2 templates.') + +MANIFEST_DIR_OPTION = click.option( + '-m', + '--manifest-dir', + 'manifest_dir', + type=click.Path(exists=True, writable=True, file_okay=False), + required=False, + help='Path to place created manifest files.') + + +@click.group(context_settings=CONTEXT_SETTINGS) +@click.option('-v', + '--verbose', + is_flag=True, + default=False, + help='Enable debug messages in log.') +def main(*, verbose): + """CLI for Airship Spyglass""" + if verbose: + log_level = logging.DEBUG + else: + log_level = logging.INFO + logging.basicConfig(format=LOG_FORMAT, level=log_level) + + +def _intermediary_helper(plugin_type, formation_data, site, excel_file, + excel_spec, additional_configuration): + LOG.info("Generating Intermediary yaml") + plugin_type = plugin_type + plugin_class = None + + # Discover the plugin and load the plugin class + LOG.info("Load the plugin class") + for entry_point in \ + pkg_resources.iter_entry_points('data_extractor_plugins'): + if entry_point.name == plugin_type: + plugin_class = entry_point.load() + + if plugin_class is None: + LOG.error( + "Unsupported Plugin type. Plugin type:{}".format(plugin_type)) + exit() + + # Extract data from plugin data source + LOG.info("Extract data from plugin data source") + data_extractor = plugin_class(site) + plugin_conf = data_extractor.get_plugin_conf({ + 'excel': excel_file, + 'excel_spec': excel_spec, + 'formation_url': formation_data[0], + 'formation_user': formation_data[1], + 'formation_password': formation_data[2] + }) # yapf: disable + data_extractor.set_config_opts(plugin_conf) + data_extractor.extract_data() + + # Apply any additional_config provided by user + additional_config = additional_configuration + if additional_config is not None: + with open(additional_config, 'r') as config: + raw_data = config.read() + additional_config_data = yaml.safe_load(raw_data) + LOG.debug("Additional config data:\n{}".format( + pprint.pformat(additional_config_data))) + + LOG.info( + "Apply additional configuration from:{}".format(additional_config)) + data_extractor.apply_additional_data(additional_config_data) + LOG.debug(pprint.pformat(data_extractor.site_data)) + + # Apply design rules to the data + LOG.info("Apply design rules to the extracted data") + process_input_ob = ProcessDataSource(site) + process_input_ob.load_extracted_data_from_data_source( + data_extractor.site_data) + return process_input_ob + + +@main.command('i', + short_help='generate intermediary', + help='Generates an intermediary file from passed excel data.') +@PLUGIN_TYPE_OPTION +@FORMATION_TARGET_OPTION +@INTERMEDIARY_DIR_OPTION +@EXCEL_FILE_OPTION +@EXCEL_SPEC_OPTION +@SITE_CONFIGURATION_FILE_OPTION +@SITE_NAME_CONFIGURATION_OPTION +def generate_intermediary(*, plugin_type, formation_target, intermediary_dir, + excel_file, excel_spec, site_configuration, + site_name): + process_input_ob = _intermediary_helper(plugin_type, formation_target, + site_name, excel_file, excel_spec, + site_configuration) + LOG.info("Generate intermediary yaml") + process_input_ob.generate_intermediary_yaml() + process_input_ob.dump_intermediary_file(intermediary_dir) + + +@main.command('m', + short_help='generates manifest and intermediary', + help='Generates manifest and intermediary files.') +@click.option( + '-i', + '--save-intermediary', + 'save_intermediary', + is_flag=True, + default=False, + help='Flag to save the generated intermediary file used for the manifests.' +) +@PLUGIN_TYPE_OPTION +@FORMATION_TARGET_OPTION +@INTERMEDIARY_DIR_OPTION +@EXCEL_FILE_OPTION +@EXCEL_SPEC_OPTION +@SITE_CONFIGURATION_FILE_OPTION +@SITE_NAME_CONFIGURATION_OPTION +@TEMPLATE_DIR_OPTION +@MANIFEST_DIR_OPTION +def generate_manifests_and_intermediary(*, save_intermediary, plugin_type, + formation_target, intermediary_dir, + excel_file, excel_spec, + site_configuration, site_name, + template_dir, manifest_dir): + process_input_ob = _intermediary_helper(plugin_type, formation_target, + site_name, excel_file, excel_spec, + site_configuration) + LOG.info("Generate intermediary yaml") + intermediary_yaml = process_input_ob.generate_intermediary_yaml() + if save_intermediary: + LOG.debug("Dumping intermediary yaml") + process_input_ob.dump_intermediary_file(intermediary_dir) + else: + LOG.debug("Skipping dump for intermediary yaml") + + LOG.info("Generating site Manifests") + processor_engine = SiteProcessor(intermediary_yaml, manifest_dir) + processor_engine.render_template(template_dir) + + +@main.command('mi', + short_help='generates manifest from intermediary', + help='Generate manifest files from specified intermediary file.') +@click.argument('intermediary_file', + type=click.Path(exists=True, readable=True, dir_okay=False)) +@TEMPLATE_DIR_OPTION +@MANIFEST_DIR_OPTION +def generate_manifests_using_intermediary(*, intermediary_file, template_dir, + manifest_dir): + LOG.info("Loading intermediary from user provided input") + with open(intermediary_file, 'r') as f: + raw_data = f.read() + intermediary_yaml = yaml.safe_load(raw_data) + + LOG.info("Generating site Manifests") + processor_engine = SiteProcessor(intermediary_yaml, manifest_dir) + processor_engine.render_template(template_dir) diff --git a/spyglass/parser/engine.py b/spyglass/parser/engine.py index 20fdeb8..0d593c2 100755 --- a/spyglass/parser/engine.py +++ b/spyglass/parser/engine.py @@ -310,7 +310,7 @@ class ProcessDataSource(object): pprint.pformat(vlan_network_data_))) def load_extracted_data_from_data_source(self, extracted_data): - """Function called from spyglass.py to pass extracted data + """Function called from cli.py to pass extracted data from input data source """ diff --git a/spyglass/spyglass.py b/spyglass/spyglass.py deleted file mode 100644 index 7b15b99..0000000 --- a/spyglass/spyglass.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright 2018 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import pprint - -import click -from pkg_resources import iter_entry_points -import yaml - -from spyglass.parser.engine import ProcessDataSource -from spyglass.site_processors.site_processor import SiteProcessor - -LOG = logging.getLogger("spyglass") - - -@click.command() -@click.option("--site", - "-s", - help="Specify the site for which manifests to be generated") -@click.option("--type", - "-t", - help="Specify the plugin type formation or tugboat") -@click.option("--formation_url", "-f", help="Specify the formation url") -@click.option("--formation_user", "-u", help="Specify the formation user id") -@click.option("--formation_password", - "-p", - help="Specify the formation user password") -@click.option( - "--intermediary", - "-i", - type=click.Path(exists=True), - help="Intermediary file path generate manifests, " - "use -m also with this option", -) -@click.option( - "--additional_config", - "-d", - type=click.Path(exists=True), - help="Site specific configuraton details", -) -@click.option( - "--generate_intermediary", - "-g", - is_flag=True, - help="Dump intermediary file from passed excel and excel spec", -) -@click.option( - "--intermediary_dir", - "-idir", - type=click.Path(exists=True), - help="The path where intermediary file needs to be generated", -) -@click.option( - "--generate_manifests", - "-m", - is_flag=True, - help="Generate manifests from the generated intermediary file", -) -@click.option( - "--manifest_dir", - "-mdir", - type=click.Path(exists=True), - help="The path where manifest files needs to be generated", -) -@click.option( - "--template_dir", - "-tdir", - type=click.Path(exists=True), - help="The path where J2 templates are available", -) -@click.option( - "--excel", - "-x", - multiple=True, - type=click.Path(exists=True), - help="Path to engineering excel file, to be passed with " - "generate_intermediary", -) -@click.option( - "--excel_spec", - "-e", - type=click.Path(exists=True), - help="Path to excel spec, to be passed with generate_intermediary", -) -@click.option( - "--loglevel", - "-l", - default=20, - multiple=False, - show_default=True, - help="Loglevel NOTSET:0 ,DEBUG:10, INFO:20, WARNING:30, ERROR:40, " - "CRITICAL:50", -) -def main(*args, **kwargs): - # Extract user provided inputs - generate_intermediary = kwargs["generate_intermediary"] - intermediary_dir = kwargs["intermediary_dir"] - generate_manifests = kwargs["generate_manifests"] - manifest_dir = kwargs["manifest_dir"] - intermediary = kwargs["intermediary"] - site = kwargs["site"] - template_dir = kwargs["template_dir"] - loglevel = kwargs["loglevel"] - - # Set Logging format - LOG.setLevel(loglevel) - stream_handle = logging.StreamHandler() - formatter = \ - logging.Formatter("(%(name)s): %(asctime)s %(levelname)s %(message)s") - stream_handle.setFormatter(formatter) - LOG.addHandler(stream_handle) - - LOG.info("Spyglass start") - LOG.info("CLI Parameters passed:\n{}".format(kwargs)) - - if not (generate_intermediary or generate_manifests): - LOG.error("Invalid CLI parameters passed!! Spyglass exited") - LOG.error("One of the options -m/-g is mandatory") - LOG.info("CLI Parameters:\n{}".format(kwargs)) - exit() - - if generate_manifests: - if template_dir is None: - LOG.error("Template directory not specified!! Spyglass exited") - LOG.error("It is mandatory to provide it when " - "generate_manifests is true") - exit() - - # Generate Intermediary yaml and manifests extracting data - # from data source specified by plugin type - if intermediary is None: - LOG.info("Generating Intermediary yaml") - plugin_type = kwargs.get("type", None) - plugin_class = None - - # Discover the plugin and load the plugin class - LOG.info("Load the plugin class") - for entry_point in iter_entry_points("data_extractor_plugins"): - if entry_point.name == plugin_type: - plugin_class = entry_point.load() - - if plugin_class is None: - LOG.error( - "Unsupported Plugin type. Plugin type:{}".format(plugin_type)) - exit() - - # Extract data from plugin data source - LOG.info("Extract data from plugin data source") - data_extractor = plugin_class(site) - plugin_conf = data_extractor.get_plugin_conf(kwargs) - data_extractor.set_config_opts(plugin_conf) - data_extractor.extract_data() - - # Apply any additional_config provided by user - additional_config = kwargs.get("additional_config", None) - if additional_config is not None: - with open(additional_config, "r") as config: - raw_data = config.read() - additional_config_data = yaml.safe_load(raw_data) - LOG.debug("Additional config data:\n{}".format( - pprint.pformat(additional_config_data))) - - LOG.info("Apply additional configuration from:{}".format( - additional_config)) - data_extractor.apply_additional_data(additional_config_data) - LOG.debug(pprint.pformat(data_extractor.site_data)) - - # Apply design rules to the data - LOG.info("Apply design rules to the extracted data") - process_input_ob = ProcessDataSource(site) - process_input_ob.load_extracted_data_from_data_source( - data_extractor.site_data) - - LOG.info("Generate intermediary yaml") - intermediary_yaml = process_input_ob.generate_intermediary_yaml() - else: - LOG.info("Loading intermediary from user provided input") - with open(intermediary, "r") as intermediary_file: - raw_data = intermediary_file.read() - intermediary_yaml = yaml.safe_load(raw_data) - - if generate_intermediary: - process_input_ob.dump_intermediary_file(intermediary_dir) - - if generate_manifests: - LOG.info("Generating site Manifests") - processor_engine = SiteProcessor(intermediary_yaml, manifest_dir) - processor_engine.render_template(template_dir) - - LOG.info("Spyglass Execution Completed") - - -if __name__ == "__main__": - main()