diff --git a/Pipfile b/Pipfile index a91a991..18865ec 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,7 @@ netaddr = "~=0.7.19" spyglass-plugin-xls = {git = "https://opendev.org/airship/spyglass-plugin-xls.git"} Jinja2 = "~=2.10.1" PyYAML = "==5.1.1" +six = "<=1.12.0" # OpenSUSE 15.1 is not compatible with versions greater than 1.12.0 [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index 7cd9f61..ab961e5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "998682b3c6b9e786e008f8d48847ec5317eb9b8054cc6cee3a91f1d57b7c5bb7" + "sha256": "af892a5802170741b6dc348ae2561538ab994416872638c3138360e1cbcf4679" }, "pipfile-spec": 6, "requires": { @@ -18,10 +18,10 @@ "default": { "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.1.0" + "version": "==19.3.0" }, "click": { "hashes": [ @@ -41,11 +41,11 @@ }, "jinja2": { "hashes": [ - "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", - "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" + "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", + "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" ], "index": "pypi", - "version": "==2.10.1" + "version": "==2.10.3" }, "jsonschema": { "hashes": [ @@ -98,9 +98,9 @@ }, "pyrsistent": { "hashes": [ - "sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" + "sha256:eb6545dbeb1aa69ab1fb4809bfbf5a8705e44d92ef8fc7c2361682a47c46c778" ], - "version": "==0.15.4" + "version": "==0.15.5" }, "pyyaml": { "hashes": [ @@ -124,11 +124,12 @@ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], + "index": "pypi", "version": "==1.12.0" }, "spyglass-plugin-xls": { "git": "https://opendev.org/airship/spyglass-plugin-xls.git", - "ref": "3b794d05ffd4731e1b7c4f23bc73a3d73f5ba1c1" + "ref": "c9d3ad98f745ad8c01bc9b9fb43a06ad63ddd112" } }, "develop": { @@ -148,10 +149,10 @@ }, "attrs": { "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.1.0" + "version": "==19.3.0" }, "bandit": { "hashes": [ @@ -200,10 +201,10 @@ }, "execnet": { "hashes": [ - "sha256:0dd40ad3b960aae93bdad7fe1c3f049bbcc8fba47094655a4301f5b33e906816", - "sha256:3839f3c1e9270926e7b3d9b0a52a57be89c302a3826a2b19c8d6e6c3d2b506d2" + "sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50", + "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547" ], - "version": "==1.7.0" + "version": "==1.7.1" }, "filelock": { "hashes": [ @@ -221,17 +222,17 @@ }, "gitdb2": { "hashes": [ - "sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2", - "sha256:e3a0141c5f2a3f635c7209d56c496ebe1ad35da82fe4d3ec4aaa36278d70648a" + "sha256:1b6df1433567a51a4a9c1a5a0de977aa351a405cc56d7d35f3388bad1f630350", + "sha256:96bbb507d765a7f51eb802554a9cfe194a174582f772e0d89f4e87288c288b7b" ], - "version": "==2.0.5" + "version": "==2.0.6" }, "gitpython": { "hashes": [ - "sha256:947cc75913e7b6da108458136607e2ee0e40c20be1e12d4284e7c6c12956c276", - "sha256:d2f4945f8260f6981d724f5957bc076398ada55cb5d25aaee10108bcdc894100" + "sha256:9c2398ffc3dcb3c40b27324b316f08a4f93ad646d5a6328cafbb871aa79f5e42", + "sha256:c155c6a2653593ccb300462f6ef533583a913e17857cfef8fc617c246b6dc245" ], - "version": "==3.0.2" + "version": "==3.0.5" }, "hacking": { "hashes": [ @@ -243,11 +244,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8", - "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3" + "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", + "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" ], "markers": "python_version < '3.8'", - "version": "==0.19" + "version": "==0.23" }, "mccabe": { "hashes": [ @@ -265,25 +266,25 @@ }, "packaging": { "hashes": [ - "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", - "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" ], - "version": "==19.1" + "version": "==19.2" }, "pbr": { "hashes": [ - "sha256:56e52299170b9492513c64be44736d27a512fa7e606f21942160b68ce510b4bc", - "sha256:9b321c204a88d8ab5082699469f52cc94c5da45c51f114113d01b3d993c24cdf" + "sha256:2c8e420cd4ed4cec4e7999ee47409e876af575d4c35a45840d59e8b5f3155ab8", + "sha256:b32c8ccaac7b1a20c0ce00ce317642e6cf231cf038f9875e0280e28af5bf7ac9" ], "index": "pypi", - "version": "==5.4.2" + "version": "==5.4.3" }, "pluggy": { "hashes": [ - "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", - "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" + "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", + "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" ], - "version": "==0.12.0" + "version": "==0.13.0" }, "py": { "hashes": [ @@ -308,41 +309,41 @@ }, "pyparsing": { "hashes": [ - "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", - "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" + "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", + "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" ], - "version": "==2.4.2" + "version": "==2.4.5" }, "pytest": { "hashes": [ - "sha256:95b1f6db806e5b1b5b443efeb58984c24945508f93a866c1719e1a507a957d7c", - "sha256:c3d5020755f70c82eceda3feaf556af9a341334414a8eca521a18f463bcead88" + "sha256:27abc3fef618a01bebb1f0d6d303d2816a99aa87a5968ebc32fe971be91eb1e6", + "sha256:58cee9e09242937e136dbb3dab466116ba20d6b7828c7620f23947f37eb4dae4" ], "index": "pypi", - "version": "==5.1.1" + "version": "==5.2.2" }, "pytest-cov": { "hashes": [ - "sha256:2b097cde81a302e1047331b48cadacf23577e431b61e9c6f49a1170bbe3d3da6", - "sha256:e00ea4fdde970725482f1f35630d12f074e121a23801aabf2ae154ec6bdd343a" + "sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b", + "sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626" ], "index": "pypi", - "version": "==2.7.1" + "version": "==2.8.1" }, "pytest-forked": { "hashes": [ - "sha256:5fe33fbd07d7b1302c95310803a5e5726a4ff7f19d5a542b7ce57c76fed8135f", - "sha256:d352aaced2ebd54d42a65825722cb433004b4446ab5d2044851d9cc7a00c9e38" + "sha256:1805699ed9c9e60cb7a8179b8d4fa2b8898098e82d229b0825d8095f0f261100", + "sha256:1ae25dba8ee2e56fb47311c9638f9e58552691da87e82d25b0ce0e4bf52b7d87" ], - "version": "==1.0.2" + "version": "==1.1.3" }, "pytest-xdist": { "hashes": [ - "sha256:3489d91516d7847db5eaecff7a2e623dba68984835dbe6cedb05ae126c4fb17f", - "sha256:501795cb99e567746f30fe78850533d4cd500c93794128e6ab9988e92a17b1f8" + "sha256:5d1b1d4461518a6023d56dab62fb63670d6f7537f23e2708459a557329accf48", + "sha256:a8569b027db70112b290911ce2ed732121876632fb3f40b1d39cd2f72f58b147" ], "index": "pypi", - "version": "==1.29.0" + "version": "==1.30.0" }, "pyyaml": { "hashes": [ @@ -366,6 +367,7 @@ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], + "index": "pypi", "version": "==1.12.0" }, "smmap2": { @@ -377,10 +379,10 @@ }, "stevedore": { "hashes": [ - "sha256:7be098ff53d87f23d798a7ce7ae5c31f094f3deb92ba18059b1aeb1ca9fec0a0", - "sha256:7d1ce610a87d26f53c087da61f06f9b7f7e552efad2a7f6d2322632b5f932ea2" + "sha256:01d9f4beecf0fbd070ddb18e5efb10567801ba7ef3ddab0074f54e3cd4e91730", + "sha256:e0739f9739a681c7a1fda76a102b65295e96a144ccdb552f2ae03c5f0abe8a14" ], - "version": "==1.30.1" + "version": "==1.31.0" }, "toml": { "hashes": [ @@ -391,18 +393,18 @@ }, "tox": { "hashes": [ - "sha256:dab0b0160dd187b654fc33d690ee1d7bf328bd5b8dc6ef3bb3cc468969c659ba", - "sha256:ee35ffce74933a6c6ac10c9a0182e41763140a5a5070e21b114feca56eaccdcd" + "sha256:1d1368ac86e8332f79e2bcef9fefe2b077469f08449eadf0183759b34f3b2070", + "sha256:bcfa3e40abc1e9b70607b56adfd976fe7dc8286ad56aab44e3151daca7d2d0d0" ], "index": "pypi", - "version": "==3.13.2" + "version": "==3.14.1" }, "virtualenv": { "hashes": [ - "sha256:94a6898293d07f84a98add34c4df900f8ec64a570292279f6d91c781d37fd305", - "sha256:f6fc312c031f2d2344f885de114f1cb029dfcffd26aa6e57d2ee2296935c4e7d" + "sha256:11cb4608930d5fd3afb545ecf8db83fa50e1f96fc4fca80c94b07d2c83146589", + "sha256:d257bb3773e48cac60e475a19b608996c73f4d333b3ba2e4e57d5ac6134e0136" ], - "version": "==16.7.4" + "version": "==16.7.7" }, "wcwidth": { "hashes": [ diff --git a/README.rst b/README.rst index 4080d62..48f5c97 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ What is Spyglass? ----------------- +----------------- Spyglass is the data extractor tool which can interface with different input data sources to generate site manifest YAML files. diff --git a/doc/source/cli.rst b/doc/source/cli.rst index 737838d..d65f295 100644 --- a/doc/source/cli.rst +++ b/doc/source/cli.rst @@ -32,6 +32,11 @@ CLI Options Enable debug logging. +Excel Plugin +************ + +Commands available under the excel plugin package. + Generate Intermediary --------------------- @@ -39,27 +44,14 @@ Generates an intermediary file from passed excel data. .. code-block:: bash - ./spyglass.sh i -p -x \ - -e -c -s + ./spyglass.sh excel intermediary -x \ + -e \ + -c \ + -s Options ^^^^^^^ -**-p / --plugin-type** "tugboat" by default. - -The plugin to use to open engineering data. Two plugins are available by -default: "tugboat" and "formation". Tugboat can be used for reading Excel data. -Formation can be used to read data from remote sources. - -**-f / --formation-target** (Required for "formation" plugin). - -Target remote for the formation plugin. Accepts a url and a username and -password to access the url. - -:: - - -f - **-d / --intermediary-dir** (Optional). Path where the intermediary file will be created. Must be a writeable @@ -80,6 +72,14 @@ engineering excel files. Must be a readable file in YAML format. Path to site specific configuration YAML. Must be a readable file. +**--intermediary-schema** (Optional). + +Path to the intermediary schema to be used for validation. + +**--no-validation** (Optional). + +Skips validation on generated intermediary data. + **-s / --site-name** (Optional). Name of the site for which the intermediary is generated. @@ -92,32 +92,17 @@ Intermediary data is always generated, but will not be saved unless specified. .. code-block:: bash - ./spyglass.sh m -t -x \ + ./spyglass.sh excel documents -x \ -e -c \ -s -t Options ^^^^^^^ -**-i / --save-intermediary** (Optional). False by default. +**-i / --generate-intermediary** (Optional). False by default. Saves the intermediary file used to make the manifests created by the command. -**-p / --plugin-type** "tugboat" by default. - -The plugin to use to open engineering data. Two plugins are available by -default: "tugboat" and "formation". Tugboat can be used for reading Excel data. -Formation can be used to read data from remote sources. - -**-f / --formation-target** (Required for "formation" plugin). - -Target remote for the formation plugin. Requires a url, a username, and a -password to access the url. - -:: - - -f - **-d / --intermediary-dir** (Optional). Path where the intermediary file will be created. Must be a writeable @@ -138,6 +123,14 @@ engineering excel files. Must be a readable file in YAML format. Path to site specific configuration YAML. Must be a readable file. +**--intermediary-schema** (Optional). + +Path to the intermediary schema to be used for validation. + +**--no-validation** (Optional). + +Skips validation on generated intermediary data. + **-s / --site-name** (Optional). Name of the site for which the intermediary is generated. @@ -152,6 +145,9 @@ Must be a readable directory with Jinja2 files using the .j2 extension. Path where generated manifest files should be written. Must be a writeable directory. +General +******* + Generate Manifests from Intermediary ------------------------------------ @@ -180,48 +176,74 @@ Must be a readable directory with Jinja2 files using the .j2 extension. **-m / --manifest-dir** (Optional). -Path where generated manifest files should be written. Must be a writeable directory. +Path where generated manifest files should be written. Must be a writeable +directory. + +**--force** (Optional). + +Forces manifests to be written, regardless of undefined data. + +Validate Documents +------------------ + +Validates pegleg documents against their schema. + +.. code-block:: bash + + spyglass validate -d -p + +Options +^^^^^^^ + +**-d / --document-path** + +Path to the document(s) to validate. + +**-p / --schema-path** + +Path to a schema or directory of schema files used to validate documents in +document path. Examples ======== Running Spyglass with Excel Plugin ----------------------------------- +********************************** .. code-block:: bash - spyglass m -i -p tugboat -x -e -c \ - -s -t + spyglass excel documents -i -x -e \ + -c -s -t Generating intermediary and manifests -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------- .. code-block:: bash - spyglass m -i -p tugboat -x SiteDesignSpec_v0.1.xlsx \ + spyglass excel documents -i -x SiteDesignSpec_v0.1.xlsx \ -e excel_spec_upstream.yaml -c site_config.yaml \ -s airship-seaworthy -t Generating intermediary without manifests -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------------------- .. code-block:: bash - spyglass i -p tugboat -x SiteDesignSpec_v0.1.xlsx \ + spyglass excel intermediary -x SiteDesignSpec_v0.1.xlsx \ -e excel_spec_upstream.yaml -c site_config.yaml \ -s airship-seaworthy Generating manifests without intermediary -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------------------- .. code-block:: bash - spyglass m -p tugboat -x SiteDesignSpec_v0.1.xlsx \ + spyglass excel documents -x SiteDesignSpec_v0.1.xlsx \ -e excel_spec_upstream.yaml -c site_config.yaml \ - -s airship-seaworthy --template_dir= + -s airship-seaworthy -t Generating manifests using intermediary -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +*************************************** .. code-block:: bash @@ -229,4 +251,11 @@ Generating manifests using intermediary Where sample 'excel_spec_upstream.yaml', 'SiteDesignSpec_v0.1.xlsx' 'site_config.yaml' and J2 templates can be found under 'spyglass/examples' -folder. \ No newline at end of file +folder. + +Validate Documents +****************** + +.. code-block:: bash + + spyglass validate -d -p diff --git a/doc/source/developer_quickstart.rst b/doc/source/developer_quickstart.rst index 1a0d90c..5f60c26 100644 --- a/doc/source/developer_quickstart.rst +++ b/doc/source/developer_quickstart.rst @@ -28,30 +28,36 @@ airship-spyglass directory. sudo apt install -y python3-pip sudo apt install -y tox -2. Set up an environment with tox. +2. Install Pipenv. + + .. code-block:: console + + pip3 install pipenv + +2. Set up an environment with Pipenv .. code-block:: console - tox -e py36 --notest + pipenv install -3. Enter the tox environment. +3. Enter the Pipenv environment. .. code-block:: console - source .tox/py36/bin/activate + pipenv shell 4. Install spyglass in the tox environment. .. code-block:: console - pip install -e . + pip3 install . 5. Run spyglass on the example files to generate an intermediate document. .. code-block:: console mkdir intermediate - spyglass m -s airship-seaworthy -p tugboat -d intermediate \ + spyglass excel documents -s airship-seaworthy -d intermediate -i \ --excel-spec spyglass/examples/excel_spec.yaml \ --excel-file spyglass/examples/SiteDesignSpec_v0.1.xlsx \ --site-configuration spyglass/examples/site_config.yaml \