diff --git a/.gitignore b/.gitignore index 6ad6eac3..18e8e67e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,6 @@ parts/ sdist/ var/ wheels/ -*.egg-info/ .installed.cfg *.egg *.tgz @@ -50,7 +49,7 @@ coverage.xml cover/* results/* .stestr/ - +.fiotest # Translations *.mo *.pot @@ -111,10 +110,21 @@ ENV/ # makefile build/lint artifacts /charts/deckhand/*.tgz /charts/deckhand/*.lock +/charts/*.tgz +/charts/*/charts +/charts/*/requirements.lock /charts/deps/*/ +/*.tgz + + # git Changelog AUTHORS # Ansible *.retry + + +# build +tmp.* +.pytest_cache \ No newline at end of file diff --git a/.zuul.yaml b/.zuul.yaml index 971dd6e4..675ac1da 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -21,44 +21,32 @@ check: jobs: - deckhand-tox-py38-postgresql - # Remove in favor of deckhand-functional-docker-py38 it runs the - # same tests just in a container - # - deckhand-functional-uwsgi-py38 + - deckhand-functional-uwsgi-py38 - deckhand-functional-docker-py38-ubuntu_focal - # non-voting, failing, incompatable with updates - # - deckhand-functional-docker-py36-ubuntu_bionic - # non-voting, failing, incompatable with updates - # Remove in favor of deckhand-uwsgi-docker-py38 it runs the - # same tests just in a container - # - deckhand-integration-uwsgi-py38 + - deckhand-integration-uwsgi-py38 - deckhand-integration-docker-py38-ubuntu_focal - # non-voting, failing, incompatable with updates - # - deckhand-integration-docker-py36-ubuntu_bionic - deckhand-chart-build-gate - deckhand-chart-build-latest-htk - deckhand-docker-build-gate-ubuntu_focal - # non-voting, failing, incompatable with updates - # - deckhand-docker-build-gate-ubuntu_bionic - openstack-tox-pep8 + # non-voting, failing gate - has to be fixed in treasuremap - deckhand-airskiff-deployment gate: jobs: - deckhand-tox-py38-postgresql + - deckhand-functional-uwsgi-py38 - deckhand-functional-docker-py38-ubuntu_focal - - deckhand-functional-docker-py36-ubuntu_bionic + - deckhand-integration-uwsgi-py38 - deckhand-integration-docker-py38-ubuntu_focal - - deckhand-integration-docker-py36-ubuntu_bionic - deckhand-chart-build-gate - deckhand-docker-build-gate-ubuntu_focal - - deckhand-docker-build-gate-ubuntu_bionic - openstack-tox-pep8 post: jobs: - deckhand-upload-git-mirror - deckhand-docker-publish-ubuntu_focal - - deckhand-docker-publish-ubuntu_bionic - deckhand-docker-tag-ubuntu_focal - - deckhand-docker-tag-ubuntu_bionic + - nodeset: name: deckhand-single-node @@ -79,10 +67,10 @@ label: ubuntu-focal - nodeset: - name: deckhand-single-node-jammy + name: deckhand-single-node-airskiff-focal nodes: - name: primary - label: ubuntu-jammy + label: ubuntu-focal - job: name: deckhand-tox-py38-postgresql @@ -95,7 +83,7 @@ - job: name: deckhand-functional-uwsgi-py38 - voting: false + voting: true description: | Run tox-based functional tests for the Airship Deckhand project using a minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf @@ -142,24 +130,10 @@ distro: ubuntu_focal irrelevant-files: *irrelevant-files -- job: - name: deckhand-functional-docker-py36-ubuntu_bionic - voting: false - description: | - Run tox-based functional tests for the Airship Deckhand project under - cPython version 3.6. Uses tox with the ``functional-py36`` environment. - Ubuntu (bionic) image is built and used. - parent: deckhand-functional-docker-base - nodeset: deckhand-single-node - vars: - tox_envlist: functional - disable_keystone: true - distro: ubuntu_bionic - irrelevant-files: *irrelevant-files - job: name: deckhand-integration-uwsgi-py38 - voting: false + voting: true description: | Run tox-based integration tests for the Airship Deckhand project using a minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf @@ -226,22 +200,11 @@ disable_keystone: false distro: ubuntu_focal -- job: - name: deckhand-integration-docker-py36-ubuntu_bionic - voting: false - description: | - Run tox-based integration tests for the Airship Deckhand project under - cPython version 3.6. Builds ubuntu (bionic) deckhand image. - parent: deckhand-integration-docker-base - nodeset: openstack-helm-single-node - vars: - disable_keystone: false - distro: ubuntu_bionic - job: name: deckhand-airskiff-deployment - voting: false - nodeset: deckhand-single-node-airskiff + voting: true + nodeset: deckhand-single-node-airskiff-focal description: | Deploy Memcached using Airskiff and submitted Deckhand changes. timeout: 9600 @@ -263,7 +226,7 @@ name: deckhand-docker-build-gate-ubuntu_focal timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node + nodeset: deckhand-single-node-focal irrelevant-files: &non-code-files-template - ^.*\.rst$ - ^doc/.*$ @@ -278,20 +241,6 @@ dynamic: patch_set: true -- job: - name: deckhand-docker-build-gate-ubuntu_bionic - voting: false - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node - irrelevant-files: *non-code-files-template - vars: - publish: false - distro: ubuntu_bionic - tags: - dynamic: - patch_set: true - - job: name: deckhand-docker-publish-ubuntu_focal description: | @@ -314,28 +263,6 @@ static: - latest -- job: - name: deckhand-docker-publish-ubuntu_bionic - voting: false - description: | - Runs on every merge, unless files in a dictionary below are changed. - Builds and publishes container ubuntu images on quay.io with a set of tags - listed in vars section. Waits in Zuul queue for a node (VM) assignment. - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: deckhand-single-node - secrets: - - airship_deckhand_quay_creds - irrelevant-files: *non-code-files-template - vars: - publish: true - distro: ubuntu_bionic - tags: - dynamic: - branch: true - commit: true - static: - - latest - job: name: deckhand-docker-tag-ubuntu_focal @@ -353,22 +280,6 @@ vars: distro: ubuntu_focal -- job: - name: deckhand-docker-tag-ubuntu_bionic - voting: false - description: | - Runs on every merge when files in a dictionalry below are changed, and - adds git commit id tag onto the ubuntu container image published on quay.io, - which has `latest` tag set. Does not wait in queue for a node (VM) - assignment, runs almost immediately. - timeout: 1800 - run: tools/gate/playbooks/docker-image-tag.yaml - nodeset: - nodes: [] - secrets: - - airship_deckhand_quay_creds - vars: - distro: ubuntu_bionic - secret: name: airship_deckhand_quay_creds diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 00000000..c2924cd0 --- /dev/null +++ b/ChangeLog @@ -0,0 +1,554 @@ +CHANGES +======= + +* [focal] Deckhand project updates +* update to focal and python 3.8 +* Allow source substring extraction +* Make failing Zuul job non-voting +* Update HTK stable commit (Ingress) +* Drop Python 3.5, make xenial/opensuse non-voting +* Helm 3: Fix Job labels +* (zuul) Fix Deckhand post jobs +* Revert jsonschema to 3.2.0 +* Deckhand gate fix +* Gate fixes +* Update pip package versions in preparation of pip 20.3 +* Accelerate YAML operations with LibYAML +* Include LibYAML in container builds +* Sort package list in Dockerfiles +* Change helm-toolkit dependency version to ">= 0.1.0" +* Fix pep8 gate running on py3.8 +* Scaling deckhand uwsgi workers +* Update HTK stable commit +* Add configmap-hash annotations for deckhand +* Implement helm-toolkit snippet to deckhand pods/containers +* [FIX] Image build checks missing setuptools +* Enabling Apparmor profile to deckhand init containers +* Remove unused code for policy validation as feature not implemented +* Re-enable all Zuul CI tests +* Add SECURITY.md +* Fix deckhand-integration-uwsgi-py35 tests +* (fix) Address uwsgi and other gating issues +* Add Docker default AppArmor profile to deckhand +* Add support for Ubuntu bionic base image +* Barbican driver simplification +* Gate fixes: pin amqp, use barbican deploy script +* Fix Deckhand integration test gates +* Remove Python 2.x support +* Fix encrypted doc rendering +* Add retries to Barbican secret create +* CI: Build image after Docker installed in airskiff +* Pin back amqp version +* Use apps/v1 k8s controllers and add labels +* CI: Remove call to deleted Airskiff script +* Allow to configure service network policy +* Let the Werkzeug package version float +* Fix for opensuse image build issue +* CI: Update Airskiff full-site manifest location +* Upgrade six to 1.12 +* Fix v2 schema support +* Add Python 3 Train unit tests +* Update packages related to requests +* Update base image from leap15.0 to leap15.1 +* Add release uuid annotation to POD spec +* Support v2 schema versions +* Add node selector to test pod +* Remove required-projects from Airskiff gate +* Add pod anti-affinity to Deckhand +* CI: Fix doc build gate +* Fixing secret name used for publishing image on quay.io +* Adding opensuse image build for deckhand +* Move nodeset to bionic +* Encrypt git mirroring ssh\_key to specific project +* Add Zuul job for mirroring to GitHub +* Make Deckhand integration tests non-voting +* Fix rtd publishing +* Docs build fix (#4) +* Docs build fix (#3) +* Docs build fix (#2) +* Docs build fix (#1) +* CI: Add Airskiff check +* CI: Update OSH relative paths for OpenDev +* OpenDev Migration Patch +* Implement Security Context for Deckhand +* Log client-id in UCP API endpoints +* CI: Add chart build jobs +* (zuul) Fix image publish post pipeline +* tools: Update Helm to v2.13.1 +* Updating Docker Gate use of zuul.newrev +* [FIX] Change Helm-toolkit pinning to new commit +* Use helm-toolkit for DB initialization +* [chart] Enable liveness probe in DH +* [ad-hoc] Update oslo.utils ver to 3.40.2 +* Embed UML generated diagrams into docs, fix docs build +* Added filename to logging message format for troubleshooting purpose +* Update oslo.util version in requirements +* schema: Fix metadata schema patterns +* Add openstack-discuss +* CI: Fix integration job +* docs(substitution): mention that all occurrences are replaced +* Add Python 3.6 classifier to setup.cfg +* Revision diffing issue with revision rollback +* Remove proxy ARG and ENV from Dockerfile +* Update url in HACKING.rst +* [FIX] Secrets substitution issue +* Fix: proper ordering: tagging after build +* fix wrong spelling +* omit the twice occured words in layering-with-replacement-single-bucket.yaml +* Create Makefile target to install Helm binary +* Minor: meaningful default label +* docs: Add use cases for each of the mutation operations +* Fix logging when "Duplicate document exists" error occurs +* fix: Use schema instead of metadata.schema for replacement check +* Validate additional 'metadata.replacement' scenarios +* Fix document is\_control method +* docs: Add config documentation to operator's section +* docs: Use sphinx-apidoc library for autodoc compatibility +* fix: Add missing requirements to doc/requirements.txt for RTD +* rtd: Fix warnings in RTD causing autodoc to fail +* requirements: Update pinned requirements +* fix: Redact secondhand substitutions of sensitive data +* Redact rendered Documents +* Fix: adding back the possibility to add arbitrary labels +* trivial: Add missing alembic upgrade head to manual install +* refactor: Move replacement checks into separate module +* docs: Add documentation on data redaction +* Redacts Raw Documents +* Validate bucket diffing works with revision rollback +* fix: Address small issues with revision rollback controller +* chore: Migrate templates from project-config to in-tree +* fix: Pin down Deckhand package requirements +* fix: Add validation logic to check for duplicate documents in engine +* Adding image tags on every commit +* docs: Elaborate on document layering in documentation +* fix: Correct .data path layering edge case +* Add explicit start/end to Deckhand response middleware +* [Gate Fix] Fix failing functional/integration tests +* optimization: Skip post-validation for rendered document cache hit +* trivial: Fix README documentation badge +* docs: Reorganize documentation structure +* Fix: various documentation and URL fixes +* Fix: git commit id labels on images +* Replace Chinese quotes with English quotes +* Adding api for revisions deep diffing +* trivial: Fix error message for non-matching policy checks +* Add release uuid to pods and rc objects (deckhand) +* Unify publishing of docs +* trivial: update description + homepage in setup.cfg +* feat(tls): add tls to ingress for public endpoint +* add python 3.6 unit test job +* substitution: Recursive pattern replacement +* Fix: Transaction rollback following DB creation error +* Fix typo +* Correct docs-on-readthedocs to work with RTD publish +* [fix] Substitution source documents accidentally modified +* trivial: Update deprecated Airship links in docs +* [Trivial Fix] Change b46enc to b64enc in chart +* Add venv tox environment +* Support rolling back to revision 0 +* Update Keystone API ports in Deckhand chart +* Chart: Use k8s secret to store config +* Implement Barbican cache for quick secret payload/ref data +* Invalidate rendered documents cache when deleting all revisions +* Remove the duplicated word +* chore(py3): update doc build to use py3 +* refactor: Clean up jsonpath\_replace method +* caching: Add test to validate shared caching across threads +* Fix typo in revision\_diff function +* Update Deckhand for latest HTK +* doc(typo): Correct spelling +* docs: Update document types documentation +* Add cryptography to Deckhand +* Implement rendered documents caching +* Remove deprecated substitution\_sources kwarg +* Rename some instances of ucp to airship +* Use concurrency to retrieve unencrypted secret data +* integration tests: Add Barbican validation/assertions +* Delete secret references from Barbican when deleting all revisions +* Move to stestr for functional/integration tests +* Fix failing integration uwsgi job +* trivial: Use airship-deckhand-single-node for nodeset in zuul.yaml +* Add test pods labels +* refactor: Use yaml.add\_representer to reduce complexity +* Move retrieval of encrypted documents to Deckhand controller +* optimization: Remove needless json.loads from middleware +* Combine integration and airship-deckhand-ubuntu jobs together +* Simplify schema validation +* Add better caching to jsonpath-ng wrapper functions +* Add integration tests job to .zuul.yaml +* Fix gate: update osh-infra-deploy-docker.yaml to align with osh +* trivial: Add orientation='reverse' to find\_cycle in layering +* layering: Support layering for primitive types +* Add functional test for validating single source multi dest substitution +* Fix gate following strange PyYAML 4.1 behavior +* Unifying proxy variables for docker build +* replacement: Fix update substitution source for replacement +* Makefile HTTP fix +* Add a readthedocs publish trigger to .zuul.yaml +* docs: Add developer overview documentation +* docs: Expand on definition of document uniqueness +* Update Deckhand test-/requirements.txt +* [test] Add integration test scenario for encrypting generic type +* chore(gate): consolidate zuul job +* trivial: Remove unused method from secrets\_manager module +* Add missing Keystone options to registration of config +* fix(gate): make the functional gate to pass +* Regression test: Validate that index >= 10 works with substitution +* [docs] Add documentation on document encryption +* Add irrelevant-files to all appropriate .zuul.yaml jobs +* fix tox python3 overrides +* (zuul) Docker image jobs +* Docker: support build behind proxy +* Allow Deckhand image to be built behind proxy +* Remove mox3 dependency +* Clean up tox.ini +* Add docs-on-readthedocs to .zuul.yaml templates +* Rename docs to doc to align with OpenStack standard +* trivial: Fix error message format +* Add py27/35 postgresql unit tests to .zuul.yaml +* style(pep8): remove identation ignores +* Zuul: Integration tests via uwsgi +* Add uwsgi functional test check to .zuul.yaml +* chore(tox): cleanup tox +* Use Ansible playbooks for functional testing gating +* Drop gather prom metrics from airship-deckhand-ubuntu job +* fix typos in documentation +* chore(image): update image +* Add functional tests to .zuul.yaml +* Update .gitreview for openstack infra +* Zuul: Initial Airship-Deckhand checks +* [fix] Parent substitution/layering before replacement +* Update Deckhand API Pod Labels +* Update Apache LICENSE +* [chart] Remove liveness probe to stop DH pod from being killed +* Add limit query filter param +* [fix gate] Fix pep8 errors +* Add no oauth middleware to bypass keystone authentication +* [fix gate] Unblock failing integration job +* [validation] Add validation codes DXXX for validation failures +* Add tests target to Makefile for Deckhand +* Add single resource substitution feeds multi destinations +* [test] Unskip integration tests +* Clean up integration test script +* Update README to correct typos and deprecated, misleading sections +* [feature] Endpoint for listing revision validations with details +* Add verbose: true to all functional tests +* [test] Cover all secret Deckhand types in integration tests +* [fix] Handles quotes in JSON path for substitution +* Make Deckhand validation exceptions adhere to UCP standard +* Add .idea/ to gitignore +* Update releasenotes/docs tox jobs +* Clean up functional test directory and entrypoint script +* Change name of Deckhand Container +* Add integration tests +* [docs] Publish releasenotes alongside docs to readthedocs +* [fix] Pass secret URI instead of UUID to barbican get\_secret +* Add negative functional test for substitution +* docs: Distinguish replace layering action from document replacement +* Fix running functional tests via uwsgi +* Raise exception on unfound secret in source document +* [fix] Drop deckhand.conf from default DECKHAND\_CONF\_DIR path +* [396582] Add alembic support to Deckhand +* [Fix] Multidigit array index +* Document replacement documentation +* [fix] Extend liveness and readiness check times +* Document replacement: Layering dependency integration +* Test that Deckhand works with YAML anchors/pointers +* Remove unused functions from DB module +* Trivial fix: Fix coverage tox.ini job +* [fix] Add uwsgi entrypoint options +* [fix] Updates to use cached jsonpath +* Enable multiple threads, disabled muliple workers +* Update kubernetes-entrypoint +* Add validation for empty documents inside multi-document payload +* [test] Improve validation policy test coverage for success scenario +* Update Makefile - Dryrun +* [TrivialFix] Unblock gate due to failing test after rebase +* Log all document data following any layering action failure +* Add functional tests for Validation Policy changes +* Add functional tests for document replacement +* Engine implementation for document replacement +* Document replacement: Update Document unique constraint +* Switch to stestr +* [398395] Update Indentation for Resource limits +* Fix secret\_uuid used to query Barbican's Secrets API +* Deprecate substitution\_sources from layering module +* Add functional test for chained substitution +* Fix uniqueness not being enforced at DB level for documents +* Skip layering for control documents +* Add readthedocs link to Deckhand readme +* Docs: Update ValidationPolicy documentation +* Trivial: Add import to base unit test to register CONF opts +* Fix: Document should not layer with parent if no layering actions +* Trivial: Rename doc to docs to align with UCP standard +* Fix condition for checking whether substitution is secret +* Fix Revision Resource print out in Deckhand client +* Deckhand API - Liveness and Readiness Probes +* Security fix: Remove document data printout from exception message +* ValidationPolicy integration with Validations API +* Improve secrets\_manager logging after 500 Internal Server Error +* Optimization: Use \_\_slots\_\_ in Deckhand engine +* Images: depreciate kolla heat-engine image for LOCI +* Add helm test to Deckhand +* Allow layering paths to include numeric indices +* Fix abstract parent documents substitutions not propagating +* Remove uwsgi.ini as it's no longer used +* Add resource declaration to deckhand job-ks-service chart template +* DH Client urls remove api/v1.0 +* Render the documents based on topological order +* Sanitize secrets contained in validation error message +* [TrivialFix] Correct regex used in jsonpath\_replace +* [TrivialFix] Fix AttributeError thrown in revision\_documents +* [TrivialFix] Log only if document parentSelector set +* Remove microversions from document versions +* Deckhand schemas as YAML files +* Update Deckhand Dockerfile +* Fix: Inject secret payload rather than reference into document +* Fix: Substitution sources not always updated during layering +* Update Makefile +* [TrivialFix] Fix BarbicanException error propagation +* Remove auto-generated AUTHORS file +* Docs: Touch up getting started documentation +* Fix Promenade: Introduce flag to only warn on missing sub source +* Add additional layering + substitution unit tests +* Docs: Update README and create Getting Started docs +* Fail fast on bad substitution input during layering +* Add label to docker image Makefile +* [client] Fix 503 exception raising attribute error instead +* Fix tox -v skipping over sqlite unit test jobs +* [Trivial Fix] Make profile directory if it doesn't exist +* Collect profile data on DH requests +* [Trivial Fix] Add document layer to error message output +* (small fix): add full path for sphinx +* Bump up package requirements versions +* Docs: Update testing documentation +* Allow unit tests to be run against in-memory sqlite +* Use DAG to resolve substitution dependency chain +* Fix: return only concrete documents from layering module +* Reduce number of pre-validation false positives +* Make layering work for grandparents not just parents +* Documentation for Exceptions +* Allow parentSelector to use multiple labels to select parent document +* [Fix gate] Fix ValueError being thrown if sub path starts with $ +* Bug Fix - DeckHand/Barbican URI Lookup +* The field returned by barbican is secret\_ref, not secret\_href +* Add missing barbican api\_endpoint to deckhand configuration +* Resolves liberal building of keystone auth parameters that end up pulling in default configuration options from the keystone\_authtoken sectiont hat are not supported by v3.Password +* Bug Fix - Update Deckhand Ingress Port +* Optimize runtime for excluding deleted documents +* Additional validation functional tests +* Fix various substitution issues +* Fix jsonpath\_replace failing to create missing array keys +* Update Deckhand README +* Fix: Make layering more performant +* Update Deckhand Chart - Database Configurability +* Improve validation error messages returned by Deckhand +* Improve secret substitution logging and look up runtime +* [TrivialFix] Un-comment-out test code in test\_revision\_diff +* Functional tests for layering + substitution scenarios +* Fix typos +* Layering edge case: Multiple empty layers +* Simplify document wrapper class +* Layering edge case: Apply substiutions to parentless document +* Allow same tag to be created for multiple revisions +* Move DB calls out of engine module into controllers +* Make the uWSGI http-timeout configurable +* Fix pifpaf not returning error code upon test failure +* Improve document validation module +* Validate correct documents used for rendering +* Sorting/filtering for rendered-documents +* Docs: Include a high-level overview of Deckhand functionality +* Update Deckhand image: logging configuration values +* Fix: Allow generic documents to be used as substitution sources +* Revert fix pifpaf run postgresql failing +* Test: add unusual documents to functional testing +* fix: Testing with multiple workers +* functional tests: Dump logs to stdout/stderr +* [Gate fix] Fix pifpaf run postgresql failing +* Remove dead validation policy code +* Update DeckHand Chart - Multi-Threads/Workers +* Update entrypoint.sh +* Simplify document layering interface +* Enable Multi-Threads in DeckHand +* Create doc/requirements.txt +* RBAC: Update serviceaccount and k8s rbac for deckhand +* Add functional tests for "owned" documents +* Test: add real-world functional schema validation +* Add blurb about using Deckhand client with Keystone Token +* DECKHAND-89: Integrate layering with rendered documents +* Test fix: remove conflicting docker run option +* DECKHAND-87: Deckhand API client library +* Functional tests via Deckhand container and Docker +* Fix up tags attribute in revisions API +* Correct recent copyright change +* [TrivialFix] Fix incorrect copyright +* Fix documentation formatting +* Fix readthedocs document build job for Deckhand +* [docs] Document schemas used for document validation +* Always rollback to the target revision +* Support filtering by schema namespace +* Implement sort filter +* Header enforcement on Content-Length 0 +* Reset primary key back to 1 after deleting all revisions +* Add expected length validation to gabbi functional tests +* Exclude previously deleted documents from current revision +* Allow anonymous access for health and versions +* Images: Remove Kolla-Toolbox image as not required +* Update to latest entrypoint container image +* DECKHAND-67: Post-rendering document validation +* Unit tests for health/versions controller +* Fix initial 'make charts' failure +* Align code with docs for validation entries +* Request middleware conditionally require content-type +* Fix corner case for document re-creation in different bucket +* Refactor unit test policy fixture +* Fix Makefile using wrong target in docker build command +* Fix rendered documents not returning all concrete documents +* Prevent same DataSchema from being used more than once for validation +* Change .to\_oslo\_conf to .to\_ini +* Deckhand Negative RBAC test scenarios +* Deckhand Makefile for CICD +* Rename Deckhand bucket endpoint to buckets for consistency +* Only allow one LayeringPolicy to exist in the system +* Create results directory for functional test results if doesn't exist +* Extended default tox testing (postgres, bandit, docs) +* Update Deckhand README and testing documentation +* HTML test report for Deckhand functional tests +* Add expected errors decorator for more resiliency +* Update DeckHand Chart +* Add health resource for ucp-integration API convention +* Make middleware enforce and validate content-type +* DECKHAND-80: Validations API Implementation +* Move Deckhand Chart +* [TrivialFix] Fix IOError being thrown by unit test +* Revamp Deckhand documentation +* Integrate Deckhand with keystone auth +* DECKHAND-66: Document substitution implementation +* Update policy and validation design documentation +* DECKHAND-61: oslo.policy integration +* Support filtering revision (documents) by any legal filter +* Add requirements for memcached +* Fix AttributeError being raised in buckets controller +* Unskip some pep8 rules +* Add releasenote management +* Fix bandit [B101:assert\_used] +* Revamp document hashing +* [tests] Downgrade postgresql to 9.5 for functional tests +* Revision rollback API +* Revision diffing API +* Deckhand postgresql compatibility +* Add sphinx job for auto-generating docs +* [flake8] Enable extra, optional hacking checks +* Clean up Deckhand 405/404 error handling +* Fix Deckhand logging +* Bucket deletion implementation +* [TrivialFix] Remove redundant requirements +* DeckHand Dockerfile +* Add basic schema validation tests +* [feat] DECKHAND-38: Secrets DB model and secrets manager +* Unskip all multi-doc CRUD functional tests +* Unskip all revision tag functional tests +* Document buckets - update logic +* Expand functional tests for revision read +* Add basic revision diffing +* [feat] DECKHAND-36 Revision tagging API +* Add rollback documentation and tests +* Initial implementation of buckets +* Add basic functional tests for substitution +* DECKHAND-33: Add oslo.config options for keystone auth +* Add concept of buckets +* Replace existing functional tests with Gabbi +* [bug] Fix response code for /POST documents if response empty +* Add bandit job to Deckhand +* [feat] DECKHAND-13: Document layering (merge) logic +* Add viewbuilder for document creation +* Add Deckhand coverage job +* Fix flake8 errors +* Add gitreview file +* [docs] Add revision tag API information to design document +* [feat] DECKHAND-28: Document pre-validation logic and API integration +* Refactor some code +* Add endpoint/tests for GET /revisions/{revision\_id} +* Fix naming conflict error +* Add view abstraction layer for modifying DB data into view data +* Raise exception instead of return +* Updated /GET revisions response body +* Remove old docstring +* Update control README (with current response bodies, even though they're a WIP +* Return YAML response body +* Add endpoint for GET /revisions +* Use built-in oslo\_db types for Columns serialized as dicts +* Finish retrieving documents by revision\_id, including with filters +* Clean up +* Test and DB API changes +* Add Revision resource +* More tests for revisions-api. Fix minor bugs +* Clarify layering actions start from full parent data +* Add DELETE endpoint +* Skip validation for abstract documents & add unit tests +* Update schema validation to be internal validation +* Update schema/db model/db api to align with design document +* Add basic RBAC details to design document +* Update documents/revisions relationship/tables +* Update revision and document tables and add more unit tests +* temp +* Revisions database and API implementation +* Update API paths for consistency +* Add clarifications based on review +* Use safe\_load\_all instead of safe\_load +* Add unit tests for db documents api +* Remove oslo\_versionedobjects +* Change application/yaml to application/x-yaml +* Cleaned up some logic, added exception handling to document creation +* Add currently necessary oslo namespaces to oslo-config-generator conf file +* Successfully creating document +* Added logic for establishing DB connection +* Refactor database sqlalchemy api/models +* Added oslo\_context-based context for oslo\_db compatibility +* Update database documents schema +* Helper for generating versioned object automatically from dictionary payload +* Add description of substitution +* Update README +* Temporary change - do not commit +* Reference Layering section in layeringDefinition description +* Add overall layering description +* Initial DB API models implementation +* Added control (API) readme +* [WIP] Implement documents API +* Add kind param to SchemaVersion class +* Change apiVersion references to schemaVersion +* Remove apiVersion attribute from substitutions.src attributes +* Remove apiVersion attribute from substitutions.src attributes +* Update default\_schema with our updated schema definition +* Trivial fix to default\_schema +* Use regexes for jsonschema pre-validation +* Add additional documentation +* Add jsonschema validation to Deckhand +* Initial engine framework +* fix typo +* Provide a separate rendered-documents endpoint +* Move reporting of validation status +* Add samples for remaining endpoints +* Address some initial review comments +* WIP: Add initial design document +* Fix incorrect comment +* Deckhand initial ORM implementation +* Deckhand initial ORM implementation +* Add kind param to SchemaVersion class +* Change apiVersion references to schemaVersion +* Remove apiVersion attribute from substitutions.src attributes +* Remove apiVersion attribute from substitutions.src attributes +* Update default\_schema with our updated schema definition +* Trivial fix to default\_schema +* Use regexes for jsonschema pre-validation +* Add additional documentation +* Add jsonschema validation to Deckhand +* Initial engine framework +* Add oslo.log integration +* DECKHAND-10: Add Barbican integration to Deckhand +* Update ChangeLog +* Update AUTHORS +* DECKHAND-2: Design core Deckhand API framework +* Oslo config integration (#1) +* Add ChangeLog +* Initial commit diff --git a/Deckhand.egg-info/PKG-INFO b/Deckhand.egg-info/PKG-INFO new file mode 100644 index 00000000..b8da56ed --- /dev/null +++ b/Deckhand.egg-info/PKG-INFO @@ -0,0 +1,101 @@ +Metadata-Version: 1.2 +Name: Deckhand +Version: 1.1.0.dev717 +Summary: Storage service for YAML-based configuration documents, which are managed through version control and automatically validated. +Home-page: https://airship-deckhand.readthedocs.io/ +Author: The Airship Authors +Author-email: airship-discuss@lists.airshipit.org +License: UNKNOWN +Description: ======== + Deckhand + ======== + + |Docker Repository on Quay| |Doc Status| + + Deckhand provides document revision management, storage and mutation + functionality upon which the rest of the `Airship`_ components rely for + orchestration of infrastructure provisioning. Deckhand understands declarative + YAML documents that define, end-to-end, the configuration of sites: from the + hardware -- encompassing network topology and hardware and host profile + information -- up to the software level that comprises the overcloud. + + * Free software: Apache license + * Documentation: https://airship-deckhand.readthedocs.io/en/latest/ + * Source: https://git.openstack.org/cgit/openstack/airship-deckhand + * Bugs: https://storyboard.openstack.org/#!/project/1004 + * Release notes: https://airship-deckhand.readthedocs.io/en/latest/releasenotes/index.html + + Core Responsibilities + ===================== + + * layering - helps reduce duplication in configuration by applying the notion + of inheritance to documents + * substitution - provides separation between secret data and other + configuration data for security purposes and reduces data duplication by + allowing common data to be defined once and substituted elsewhere dynamically + * revision history - maintains well-defined collections of documents within + immutable revisions that are meant to operate together, while providing the + ability to rollback to previous revisions + * validation - allows services to implement and register different kinds of + validations and report errors + * secret management - leverages existing OpenStack APIs -- namely + `Barbican`_ -- to reliably and securely store sensitive data + + .. _Barbican: https://docs.openstack.org/barbican/latest/api/ + + Getting Started + =============== + + For more detailed installation and setup information, please refer to the + `Getting Started `_ + guide. + + Integration Points + ================== + + Deckhand has the following integration points: + + * `Barbican (OpenStack Key Manager) `_ + provides secure storage for sensitive data. + * `Keystone (OpenStack Identity service) `_ + provides authentication and support for role based authorization. + * `PostgreSQL `_ is used to persist information + to correlate workflows with users and history of workflow commands. + + .. note:: + + Currently, other database back-ends are not supported. + + Though, being a low-level service, has many other Airship services that integrate + with it, including: + + * `Drydock `_ is orchestrated by + Shipyard to perform bare metal node provisioning. + * `Promenade `_ is indirectly + orchestrated by Shipyard to configure and join Kubernetes nodes. + * `Armada `_ is orchestrated by + Shipyard to deploy and test Kubernetes workloads. + + Further Reading + =============== + + `Airship`_. + + .. _Airship: https://www.airshipit.org + + .. |Docker Repository on Quay| image:: https://quay.io/repository/airshipit/deckhand/status + :target: https://quay.io/repository/airshipit/deckhand + .. |Doc Status| image:: https://readthedocs.org/projects/airship-deckhand/badge/?version=latest + :target: https://airship-deckhand.readthedocs.io/ + + +Platform: UNKNOWN +Classifier: Intended Audience :: Information Technology +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: POSIX :: Linux +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.10 +Requires-Python: >=3.8 diff --git a/Deckhand.egg-info/SOURCES.txt b/Deckhand.egg-info/SOURCES.txt new file mode 100644 index 00000000..4c0d5210 --- /dev/null +++ b/Deckhand.egg-info/SOURCES.txt @@ -0,0 +1,388 @@ +.coveragerc +.dockerignore +.stestr.conf +.zuul.yaml +AUTHORS +ChangeLog +HACKING.rst +LICENSE +Makefile +README.rst +REVIEWING.rst +alembic.ini +bindep.txt +entrypoint.sh +requirements-direct.txt +requirements-frozen.txt +requirements.txt +setup.cfg +setup.py +test-requirements.txt +tox.ini +.github/SECURITY.md +Deckhand.egg-info/PKG-INFO +Deckhand.egg-info/SOURCES.txt +Deckhand.egg-info/dependency_links.txt +Deckhand.egg-info/entry_points.txt +Deckhand.egg-info/not-zip-safe +Deckhand.egg-info/pbr.json +Deckhand.egg-info/requires.txt +Deckhand.egg-info/top_level.txt +alembic/README +alembic/env.py +alembic/script.py.mako +alembic/versions/918bbfd28185_initial_deckhand_base.py +charts/deckhand/.helmignore +charts/deckhand/Chart.yaml +charts/deckhand/requirements.yaml +charts/deckhand/values.yaml +charts/deckhand/templates/configmap-bin.yaml +charts/deckhand/templates/configmap-etc.yaml +charts/deckhand/templates/deployment.yaml +charts/deckhand/templates/ingress-api.yaml +charts/deckhand/templates/job-db-init.yaml +charts/deckhand/templates/job-db-sync.yaml +charts/deckhand/templates/job-image-repo-sync.yaml +charts/deckhand/templates/job-ks-endpoints.yaml +charts/deckhand/templates/job-ks-service.yaml +charts/deckhand/templates/job-ks-user.yaml +charts/deckhand/templates/network_policy.yaml +charts/deckhand/templates/secret-db.yaml +charts/deckhand/templates/secret-ingress-tls.yaml +charts/deckhand/templates/secret-keystone-env.yaml +charts/deckhand/templates/service-ingress.yaml +charts/deckhand/templates/service.yaml +charts/deckhand/templates/bin/_db-sync.sh.tpl +charts/deckhand/templates/tests/test-deckhand-api.yaml +charts/deps/.gitkeep +deckhand/__init__.py +deckhand/cmd.py +deckhand/context.py +deckhand/errors.py +deckhand/factories.py +deckhand/policy.py +deckhand/service.py +deckhand/types.py +deckhand/Deckhand.egg-info/PKG-INFO +deckhand/Deckhand.egg-info/SOURCES.txt +deckhand/Deckhand.egg-info/dependency_links.txt +deckhand/Deckhand.egg-info/entry_points.txt +deckhand/Deckhand.egg-info/not-zip-safe +deckhand/Deckhand.egg-info/pbr.json +deckhand/Deckhand.egg-info/requires.txt +deckhand/Deckhand.egg-info/top_level.txt +deckhand/barbican/__init__.py +deckhand/barbican/cache.py +deckhand/barbican/client_wrapper.py +deckhand/barbican/driver.py +deckhand/client/__init__.py +deckhand/client/base.py +deckhand/client/buckets.py +deckhand/client/client.py +deckhand/client/exceptions.py +deckhand/client/revisions.py +deckhand/client/tags.py +deckhand/common/__init__.py +deckhand/common/document.py +deckhand/common/utils.py +deckhand/common/validation_message.py +deckhand/conf/__init__.py +deckhand/conf/config.py +deckhand/conf/opts.py +deckhand/control/__init__.py +deckhand/control/api.py +deckhand/control/base.py +deckhand/control/buckets.py +deckhand/control/common.py +deckhand/control/health.py +deckhand/control/middleware.py +deckhand/control/no_oauth_middleware.py +deckhand/control/revision_deepdiffing.py +deckhand/control/revision_diffing.py +deckhand/control/revision_documents.py +deckhand/control/revision_tags.py +deckhand/control/revisions.py +deckhand/control/rollback.py +deckhand/control/validations.py +deckhand/control/versions.py +deckhand/control/views/__init__.py +deckhand/control/views/document.py +deckhand/control/views/revision.py +deckhand/control/views/revision_tag.py +deckhand/control/views/validation.py +deckhand/db/__init__.py +deckhand/db/sqlalchemy/__init__.py +deckhand/db/sqlalchemy/api.py +deckhand/db/sqlalchemy/models.py +deckhand/engine/__init__.py +deckhand/engine/_replacement.py +deckhand/engine/cache.py +deckhand/engine/document_validation.py +deckhand/engine/layering.py +deckhand/engine/render.py +deckhand/engine/revision_diff.py +deckhand/engine/secrets_manager.py +deckhand/engine/utils.py +deckhand/engine/schemas/base_schema.yaml +deckhand/engine/schemas/certificate_authority_key_schema.yaml +deckhand/engine/schemas/certificate_authority_schema.yaml +deckhand/engine/schemas/certificate_key_schema.yaml +deckhand/engine/schemas/certificate_schema.yaml +deckhand/engine/schemas/layering_policy_schema.yaml +deckhand/engine/schemas/metadata_control.yaml +deckhand/engine/schemas/metadata_document.yaml +deckhand/engine/schemas/passphrase_schema.yaml +deckhand/engine/schemas/private_key_schema.yaml +deckhand/engine/schemas/public_key_schema.yaml +deckhand/engine/schemas/validation_policy_schema.yaml +deckhand/policies/__init__.py +deckhand/policies/base.py +deckhand/policies/document.py +deckhand/policies/revision.py +deckhand/policies/revision_tag.py +deckhand/policies/validation.py +deckhand/tests/__init__.py +deckhand/tests/deckhand.conf.test +deckhand/tests/test_utils.py +deckhand/tests/common/__init__.py +deckhand/tests/common/test_gabbi.py +deckhand/tests/functional/README.rst +deckhand/tests/functional/gabbits/document/document-crud-error-bucket-conflict.yaml +deckhand/tests/functional/gabbits/document/document-crud-success-multi-bucket.yaml +deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml +deckhand/tests/functional/gabbits/document/document-crud-success-single-bucket.yaml +deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml +deckhand/tests/functional/gabbits/layering/layering-multiple-bucket.yaml +deckhand/tests/functional/gabbits/layering/layering-policy-conflict.yaml +deckhand/tests/functional/gabbits/layering/layering-single-bucket.yaml +deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml +deckhand/tests/functional/gabbits/layering/layering-with-substitution-single-bucket.yaml +deckhand/tests/functional/gabbits/replacement/multi-layer-replacement.yaml +deckhand/tests/functional/gabbits/replacement/replacement.yaml +deckhand/tests/functional/gabbits/resources/chained-substitution.yaml +deckhand/tests/functional/gabbits/resources/deckhand-owned-sample.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-2-layers.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-3-layers.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-a.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-b.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-delete.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-update.yaml +deckhand/tests/functional/gabbits/resources/design-doc-substitution-generic-sample.yaml +deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml +deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml +deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml +deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml +deckhand/tests/functional/gabbits/resources/layering-and-substitution-sample.yaml +deckhand/tests/functional/gabbits/resources/layering-needs-substitution-source.yaml +deckhand/tests/functional/gabbits/resources/passphrase.yaml +deckhand/tests/functional/gabbits/resources/replacement.yaml +deckhand/tests/functional/gabbits/resources/sample-doc.yaml +deckhand/tests/functional/gabbits/resources/sample-schema-v2.yaml +deckhand/tests/functional/gabbits/resources/sample-schema.yaml +deckhand/tests/functional/gabbits/resources/sample-tag-data.yaml +deckhand/tests/functional/gabbits/resources/substitution-results-in-none-bug.yaml +deckhand/tests/functional/gabbits/resources/ucp-sample-documents.yaml +deckhand/tests/functional/gabbits/resources/unusual-documents.yaml +deckhand/tests/functional/gabbits/revision/revision-crud-success-single-bucket.yaml +deckhand/tests/functional/gabbits/revision/revision-filters.yaml +deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml +deckhand/tests/functional/gabbits/revision-diff/revision-diff-rollback-null-success.yaml +deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml +deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters-negative.yaml +deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters.yaml +deckhand/tests/functional/gabbits/revision-documents/revision-documents-multiple-filters.yaml +deckhand/tests/functional/gabbits/revision-rollback/rollback-success-single-bucket.yaml +deckhand/tests/functional/gabbits/revision-tag/revision-tag-success.yaml +deckhand/tests/functional/gabbits/substitution/substitution-chained-single-bucket.yaml +deckhand/tests/functional/gabbits/substitution/substitution-multiple-bucket.yaml +deckhand/tests/functional/gabbits/substitution/substitution-results-in-none-bug.yaml +deckhand/tests/functional/gabbits/substitution/substitution-single-bucket-generic.yaml +deckhand/tests/functional/gabbits/substitution/substitution-single-bucket.yaml +deckhand/tests/functional/gabbits/substitution/substitution-source-feeds-multi-dest.yaml +deckhand/tests/integration/README.rst +deckhand/tests/integration/__init__.py +deckhand/tests/integration/gabbits/document-crud-secret.yaml +deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml +deckhand/tests/integration/gabbits/document-render-secret.yaml +deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml +deckhand/tests/integration/gabbits/document-substitution-secret.yaml +deckhand/tests/integration/gabbits/revision-delete-all-secrets.yaml +deckhand/tests/unit/__init__.py +deckhand/tests/unit/base.py +deckhand/tests/unit/dh_fixtures.py +deckhand/tests/unit/fake_policy.py +deckhand/tests/unit/test_policy.py +deckhand/tests/unit/barbican/__init__.py +deckhand/tests/unit/barbican/test_cache.py +deckhand/tests/unit/common/__init__.py +deckhand/tests/unit/common/test_utils.py +deckhand/tests/unit/control/__init__.py +deckhand/tests/unit/control/base.py +deckhand/tests/unit/control/test_api_initialization.py +deckhand/tests/unit/control/test_base_controller.py +deckhand/tests/unit/control/test_buckets_controller.py +deckhand/tests/unit/control/test_errors.py +deckhand/tests/unit/control/test_health_controller.py +deckhand/tests/unit/control/test_middleware.py +deckhand/tests/unit/control/test_rendered_documents_controller.py +deckhand/tests/unit/control/test_revision_documents_controller.py +deckhand/tests/unit/control/test_revision_tags_controller.py +deckhand/tests/unit/control/test_revisions_controller.py +deckhand/tests/unit/control/test_revisions_deepdiff_controller.py +deckhand/tests/unit/control/test_revisions_diff_controller.py +deckhand/tests/unit/control/test_revisions_rollback_controller.py +deckhand/tests/unit/control/test_versions_controller.py +deckhand/tests/unit/db/__init__.py +deckhand/tests/unit/db/test_documents.py +deckhand/tests/unit/db/test_documents_negative.py +deckhand/tests/unit/db/test_layering_policies.py +deckhand/tests/unit/db/test_revision_documents.py +deckhand/tests/unit/db/test_revision_rollback.py +deckhand/tests/unit/db/test_revision_tags.py +deckhand/tests/unit/db/test_revision_tags_negative.py +deckhand/tests/unit/db/test_revisions.py +deckhand/tests/unit/engine/__init__.py +deckhand/tests/unit/engine/base.py +deckhand/tests/unit/engine/test_cache.py +deckhand/tests/unit/engine/test_document_layering.py +deckhand/tests/unit/engine/test_document_layering_and_replacement.py +deckhand/tests/unit/engine/test_document_layering_and_replacement_negative.py +deckhand/tests/unit/engine/test_document_layering_and_substitution.py +deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py +deckhand/tests/unit/engine/test_document_layering_negative.py +deckhand/tests/unit/engine/test_document_validation.py +deckhand/tests/unit/engine/test_document_validation_negative.py +deckhand/tests/unit/engine/test_revision_deepdiffing.py +deckhand/tests/unit/engine/test_revision_diffing.py +deckhand/tests/unit/engine/test_secrets_manager.py +deckhand/tests/unit/resources/sample_certificate.yaml +deckhand/tests/unit/resources/sample_certificate_authority.yaml +deckhand/tests/unit/resources/sample_certificate_authority_key.yaml +deckhand/tests/unit/resources/sample_certificate_key.yaml +deckhand/tests/unit/resources/sample_data_schema.yaml +deckhand/tests/unit/resources/sample_document.yaml +deckhand/tests/unit/resources/sample_document_simple.yaml +deckhand/tests/unit/resources/sample_layering_policy.yaml +deckhand/tests/unit/resources/sample_passphrase.yaml +deckhand/tests/unit/resources/sample_private_key.yaml +deckhand/tests/unit/resources/sample_public_key.yaml +deckhand/tests/unit/resources/sample_validation_policy.yaml +deckhand/tests/unit/views/__init__.py +deckhand/tests/unit/views/test_document_views.py +deckhand/tests/unit/views/test_revision_tag_views.py +deckhand/tests/unit/views/test_revision_views.py +doc/.gitignore +doc/requirements-docs.txt +doc/source/conf.py +doc/source/glossary.rst +doc/source/index.rst +doc/source/overview.rst +doc/source/releasenotes +doc/source/_static/.placeholder +doc/source/contributor/HACKING.rst +doc/source/contributor/REVIEWING.rst +doc/source/contributor/developer-overview.rst +doc/source/contributor/index.rst +doc/source/contributor/policy-enforcement.rst +doc/source/contributor/testing.rst +doc/source/diagrams/architecture-pegleg.uml +doc/source/diagrams/architecture.uml +doc/source/operators/api_client.rst +doc/source/operators/api_ref.rst +doc/source/operators/configuration.rst +doc/source/operators/exceptions.rst +doc/source/operators/index.rst +doc/source/operators/multi-distro-support.rst +doc/source/users/document-types.rst +doc/source/users/documents.rst +doc/source/users/encryption.rst +doc/source/users/getting-started.rst +doc/source/users/index.rst +doc/source/users/layering.rst +doc/source/users/rendering.rst +doc/source/users/replacement.rst +doc/source/users/revision-history.rst +doc/source/users/substitution.rst +doc/source/users/validation.rst +etc/deckhand/config-generator.conf +etc/deckhand/deckhand-paste.ini +etc/deckhand/deckhand.conf.sample +etc/deckhand/logging.conf.sample +etc/deckhand/noauth-paste.ini +etc/deckhand/policy-generator.conf +etc/deckhand/policy.yaml.sample +images/deckhand/Dockerfile.opensuse_15 +images/deckhand/Dockerfile.ubuntu_bionic +images/deckhand/Dockerfile.ubuntu_focal +releasenotes/notes/development-mode-51208c39e9eee34f.yaml +releasenotes/notes/fix-attribute-error-in-bucket-controller-b5c9e410823abd19.yaml +releasenotes/notes/fix-unique-layering-policy-bac2940c3deeba51.yaml +releasenotes/notes/list-validations-details-endpoint-c3f18963b1372e40.yaml +releasenotes/notes/oslo.policy-integration-f03ac6a7a2ccef5a.yaml +releasenotes/notes/revision-document-filtering-0e57274a4fc1bc07.yaml +releasenotes/notes/rm-pep8-ignores-974a114e488ff1ea.yaml +releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml +releasenotes/notes/validations-api-cf07c8b6b5040f67.yaml +releasenotes/source/conf.py +releasenotes/source/index.rst +releasenotes/source/unreleased.rst +releasenotes/source/_static/.placeholder +releasenotes/source/_templates/.placeholder +tools/build-docs.sh +tools/common-tests.sh +tools/functional-tests.sh +tools/helm_install.sh +tools/helm_tk.sh +tools/image_tags.py +tools/integration-tests.sh +tools/run_pifpaf.sh +tools/unit-tests.sh +tools/whitespace-linter.sh +tools/gate/playbooks/airskiff-deploy.yaml +tools/gate/playbooks/airskiff-reduce-site.yaml +tools/gate/playbooks/build-charts.yaml +tools/gate/playbooks/debug-report.yaml +tools/gate/playbooks/docker-image-build.yaml +tools/gate/playbooks/docker-image-tag.yaml +tools/gate/playbooks/git-config.yaml +tools/gate/playbooks/install-postgresql.yaml +tools/gate/playbooks/osh-infra-build.yaml +tools/gate/playbooks/osh-infra-collect-logs.yaml +tools/gate/playbooks/osh-infra-deploy-docker.yaml +tools/gate/playbooks/osh-infra-deploy-k8s.yaml +tools/gate/playbooks/osh-infra-upgrade-host.yaml +tools/gate/playbooks/roles +tools/gate/playbooks/run-functional-tests-docker.yaml +tools/gate/playbooks/run-functional-tests-uwsgi.yaml +tools/gate/playbooks/run-integration-tests-docker.yaml +tools/gate/playbooks/run-integration-tests-uwsgi.yaml +tools/gate/playbooks/vars.yaml +tools/gate/roles/build-charts/tasks/build-charts.yaml +tools/gate/roles/build-charts/tasks/main.yaml +tools/gate/roles/build-images/defaults/main.yaml +tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml +tools/gate/roles/build-images/tasks/main.yaml +tools/gate/roles/deploy-apparmor/tasks/main.yaml +tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml +tools/gate/roles/deploy-barbican/tasks/main.yaml +tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml +tools/gate/roles/deploy-deckhand/tasks/main.yaml +tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml +tools/gate/roles/deploy-keystone-dependencies/tasks/main.yaml +tools/gate/roles/deploy-postgresql/tasks/deploy-postgresql.yaml +tools/gate/roles/deploy-postgresql/tasks/main.yaml +tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml +tools/gate/roles/disable-systemd-resolved/tasks/main.yaml +tools/gate/roles/generate-test-config/tasks/generate-test-config.yaml +tools/gate/roles/generate-test-config/tasks/generate-test-paste.yaml +tools/gate/roles/generate-test-config/tasks/main.yaml +tools/gate/roles/install-postgresql/tasks/install-postgresql.yaml +tools/gate/roles/install-postgresql/tasks/main.yaml +tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml +tools/gate/roles/install-test-requirements/tasks/main.yaml +tools/gate/roles/run-functional-tests/tasks/functional-tests.yaml +tools/gate/roles/run-functional-tests/tasks/main.yaml +tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml +tools/gate/roles/run-integration-tests/tasks/main.yaml +tools/gate/scripts/010-build-charts.sh +tools/gate/scripts/020-deploy-postgresql.sh \ No newline at end of file diff --git a/Deckhand.egg-info/dependency_links.txt b/Deckhand.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/Deckhand.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/Deckhand.egg-info/entry_points.txt b/Deckhand.egg-info/entry_points.txt new file mode 100644 index 00000000..4c1b5448 --- /dev/null +++ b/Deckhand.egg-info/entry_points.txt @@ -0,0 +1,6 @@ +[oslo.config.opts] +deckhand.conf = deckhand.conf.opts:list_opts + +[oslo.policy.policies] +deckhand = deckhand.policies:list_rules + diff --git a/Deckhand.egg-info/not-zip-safe b/Deckhand.egg-info/not-zip-safe new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/Deckhand.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/Deckhand.egg-info/pbr.json b/Deckhand.egg-info/pbr.json new file mode 100644 index 00000000..754cc7c5 --- /dev/null +++ b/Deckhand.egg-info/pbr.json @@ -0,0 +1 @@ +{"git_version": "c942ba7", "is_release": false} \ No newline at end of file diff --git a/Deckhand.egg-info/requires.txt b/Deckhand.egg-info/requires.txt new file mode 100644 index 00000000..44ab493f --- /dev/null +++ b/Deckhand.egg-info/requires.txt @@ -0,0 +1,77 @@ +alembic<=1.4.3 +amqp==2.6.1 +Beaker<=1.12.0 +chardet==3.0.4 +charset_normalizer<4.0.0,>=2.0.0 +ConfigParser +coverage +cryptography==3.4.8 +deepdiff<=5.8.1 +docutils +falcon +hacking +importlib-metadata +Jinja2 +jsonpath_ng +jsonpath-rw-ext>=1.0.0 +jsonpickle==1.4.1 +jsonschema<=3.2.0 +keystoneauth1<=5.1.1 +keystonemiddleware +kombu<=4.6.11 +MarkupSafe<2.1.0,>=0.9.2 +networkx +nose +oslo.cache<=2.10.1 +oslo.concurrency +oslo.config<=8.7.1 +oslo.context<=4.1.0 +oslo.db<=10.0.0 +oslo.log<=4.6.0 +oslo.messaging<=12.13.0 +oslo.middleware<=4.4.0 +oslo.policy<=3.10.1 +oslo.serialization<=4.2.0 +oslo.utils<=4.12.3 +packaging==21.3 +Paste<=3.5.0 +PasteDeploy +PasteScript +pbr<=5.5.1 +psycopg2-binary +pycadf<=3.1.1 +pycodestyle<=2.6.0 +pyflakes<=2.2.0 +Pygments<=2.14.0 +pylibyaml==0.1.0 +pymongo +pyparsing<=2.4.7 +pyproject-api +python-barbicanclient<=5.2.0 +python-dateutil +python-keystoneclient +python-memcached +python-subunit<=1.4.0 +PyYAML<=5.4.1 +reno +requests==2.27.0 +Routes +setuptools<=45.2.0 +six +Sphinx +sphinx-rtd-theme==0.5.0 +SQLAlchemy<=1.3.20 +stevedore +testrepository +testresources +testscenarios +testtools<=2.5.0 +tiddlyweb +typing_extensions==4.5.0 +urllib3<=1.25.11,>=1.21.1 +uWSGI==2.0.21 +virtualenv +Werkzeug +wheel +wsgi-intercept>=1.2.2 +yq diff --git a/Deckhand.egg-info/top_level.txt b/Deckhand.egg-info/top_level.txt new file mode 100644 index 00000000..1687a967 --- /dev/null +++ b/Deckhand.egg-info/top_level.txt @@ -0,0 +1 @@ +deckhand diff --git a/HACKING.rst b/HACKING.rst index 80c7a6d9..728c0f7e 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -45,7 +45,7 @@ without the patch and passes with the patch. Running Tests ------------- -The testing system is based on a combination of tox and testr. The canonical +The testing system is based on a combination of tox and stestr. The canonical approach to running tests is to simply run the command ``tox``. This will create virtual environments, populate them with dependencies and run all of the tests that OpenStack CI systems run. Behind the scenes, tox is running diff --git a/Makefile b/Makefile index c3b7f280..6da5acfe 100644 --- a/Makefile +++ b/Makefile @@ -37,14 +37,10 @@ images: build_deckhand # Create tgz of the chart .PHONY: charts -charts: helm-toolkit +charts: helm-init $(HELM) dep up charts/deckhand $(HELM) package charts/deckhand -# Initialize local helm config -.PHONY: helm-toolkit -helm-toolkit: helm-install - tools/helm_tk.sh $(HELM) # Install helm binary .PHONY: helm-install @@ -57,8 +53,8 @@ lint: pep8 helm_lint # Dry run templating of chart .PHONY: dry-run -dry-run: clean - tools/helm_tk.sh $(HELM) +dry-run: helm-init + $(HELM) dep up charts/deckhand $(HELM) template charts/deckhand .PHONY: tests @@ -115,6 +111,10 @@ pep8: tox -e pep8 .PHONY: helm_lint -helm_lint: clean - tools/helm_tk.sh $(HELM) +helm_lint: helm-init + $(HELM) dep up charts/deckhand $(HELM) lint charts/deckhand + +# Initialize local helm config +helm-init: helm-install + tools/helm_tk.sh $(HELM) \ No newline at end of file diff --git a/alembic/versions/918bbfd28185_initial_deckhand_base.py b/alembic/versions/918bbfd28185_initial_deckhand_base.py index 7fb0129d..ff021a15 100644 --- a/alembic/versions/918bbfd28185_initial_deckhand_base.py +++ b/alembic/versions/918bbfd28185_initial_deckhand_base.py @@ -6,6 +6,7 @@ Create Date: 2018-04-04 17:19:24.222703 """ import logging +import six from alembic import op import sqlalchemy as sa @@ -70,7 +71,7 @@ def upgrade(): LOG.info("Finding tables with query: %s", tables_select) rs = conn.execute(tables_select) existing_tables = [row[0] for row in rs] - LOG.info("Existing tables: %s", str(existing_tables)) + LOG.info("Existing tables: %s", six.text_type(existing_tables)) if 'buckets' not in existing_tables: LOG.info("'buckets' not present. Creating table") @@ -131,7 +132,7 @@ def upgrade(): check_documents_columns) rs = conn.execute(check_documents_columns) columns = [row[0] for row in rs] - LOG.info("Columns are: %s", str(columns)) + LOG.info("Columns are: %s", six.text_type(columns)) if '_metadata' in columns: LOG.info("Found '_metadata' column; will rename to 'meta'") diff --git a/bindep.txt b/bindep.txt index b8ef165e..fdcc7d7d 100644 --- a/bindep.txt +++ b/bindep.txt @@ -4,3 +4,11 @@ # PlantUML is used for documentation builds, graphviz is it's soft dependancy plantuml graphviz +libffi-dev [test platform:dpkg] +libkrb5-dev [platform:dpkg] +libpq-dev [platform:dpkg] +libsasl2-dev [platform:dpkg] +libssl-dev [platform:dpkg] +libre2-dev [platform:dpkg] +postgresql [platform:dpkg] +ethtool [platform:dpkg] diff --git a/charts/deckhand/Chart.yaml b/charts/deckhand/Chart.yaml index 298bc7ed..ad0a0871 100644 --- a/charts/deckhand/Chart.yaml +++ b/charts/deckhand/Chart.yaml @@ -15,8 +15,8 @@ apiVersion: v1 description: A Helm chart for Deckhand name: deckhand -version: 0.2.0 -appVersion: 1.0 +version: 0.2.2 +appVersion: 1.1.0 keywords: - deckhand home: https://github.com/openstack/airship-deckhand diff --git a/charts/deckhand/values.yaml b/charts/deckhand/values.yaml index 431203d7..88a406be 100644 --- a/charts/deckhand/values.yaml +++ b/charts/deckhand/values.yaml @@ -27,14 +27,14 @@ labels: images: tags: - deckhand: quay.io/airshipit/deckhand:latest - dep_check: "quay.io/stackanetes/kubernetes-entrypoint:v0.3.1" - db_init: docker.io/postgres:9.5 - db_sync: quay.io/airshipit/deckhand:latest - image_repo_sync: docker.io/docker:17.07.0 - ks_endpoints: docker.io/openstackhelm/heat:newton - ks_service: docker.io/openstackhelm/heat:newton - ks_user: docker.io/openstackhelm/heat:newton + deckhand: quay.io/airshipit/deckhand:latestlatest-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 + db_init: docker.io/postgres:14.6 + db_sync: quay.io/airshipit/deckhand:latest-ubuntu_focal + image_repo_sync: docker.io/docker:23.0.3 + ks_endpoints: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + ks_service: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal pull_policy: "IfNotPresent" local_registry: active: false diff --git a/deckhand/Deckhand.egg-info/PKG-INFO b/deckhand/Deckhand.egg-info/PKG-INFO new file mode 100644 index 00000000..b8da56ed --- /dev/null +++ b/deckhand/Deckhand.egg-info/PKG-INFO @@ -0,0 +1,101 @@ +Metadata-Version: 1.2 +Name: Deckhand +Version: 1.1.0.dev717 +Summary: Storage service for YAML-based configuration documents, which are managed through version control and automatically validated. +Home-page: https://airship-deckhand.readthedocs.io/ +Author: The Airship Authors +Author-email: airship-discuss@lists.airshipit.org +License: UNKNOWN +Description: ======== + Deckhand + ======== + + |Docker Repository on Quay| |Doc Status| + + Deckhand provides document revision management, storage and mutation + functionality upon which the rest of the `Airship`_ components rely for + orchestration of infrastructure provisioning. Deckhand understands declarative + YAML documents that define, end-to-end, the configuration of sites: from the + hardware -- encompassing network topology and hardware and host profile + information -- up to the software level that comprises the overcloud. + + * Free software: Apache license + * Documentation: https://airship-deckhand.readthedocs.io/en/latest/ + * Source: https://git.openstack.org/cgit/openstack/airship-deckhand + * Bugs: https://storyboard.openstack.org/#!/project/1004 + * Release notes: https://airship-deckhand.readthedocs.io/en/latest/releasenotes/index.html + + Core Responsibilities + ===================== + + * layering - helps reduce duplication in configuration by applying the notion + of inheritance to documents + * substitution - provides separation between secret data and other + configuration data for security purposes and reduces data duplication by + allowing common data to be defined once and substituted elsewhere dynamically + * revision history - maintains well-defined collections of documents within + immutable revisions that are meant to operate together, while providing the + ability to rollback to previous revisions + * validation - allows services to implement and register different kinds of + validations and report errors + * secret management - leverages existing OpenStack APIs -- namely + `Barbican`_ -- to reliably and securely store sensitive data + + .. _Barbican: https://docs.openstack.org/barbican/latest/api/ + + Getting Started + =============== + + For more detailed installation and setup information, please refer to the + `Getting Started `_ + guide. + + Integration Points + ================== + + Deckhand has the following integration points: + + * `Barbican (OpenStack Key Manager) `_ + provides secure storage for sensitive data. + * `Keystone (OpenStack Identity service) `_ + provides authentication and support for role based authorization. + * `PostgreSQL `_ is used to persist information + to correlate workflows with users and history of workflow commands. + + .. note:: + + Currently, other database back-ends are not supported. + + Though, being a low-level service, has many other Airship services that integrate + with it, including: + + * `Drydock `_ is orchestrated by + Shipyard to perform bare metal node provisioning. + * `Promenade `_ is indirectly + orchestrated by Shipyard to configure and join Kubernetes nodes. + * `Armada `_ is orchestrated by + Shipyard to deploy and test Kubernetes workloads. + + Further Reading + =============== + + `Airship`_. + + .. _Airship: https://www.airshipit.org + + .. |Docker Repository on Quay| image:: https://quay.io/repository/airshipit/deckhand/status + :target: https://quay.io/repository/airshipit/deckhand + .. |Doc Status| image:: https://readthedocs.org/projects/airship-deckhand/badge/?version=latest + :target: https://airship-deckhand.readthedocs.io/ + + +Platform: UNKNOWN +Classifier: Intended Audience :: Information Technology +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: POSIX :: Linux +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.10 +Requires-Python: >=3.8 diff --git a/deckhand/Deckhand.egg-info/SOURCES.txt b/deckhand/Deckhand.egg-info/SOURCES.txt new file mode 100644 index 00000000..ff6767da --- /dev/null +++ b/deckhand/Deckhand.egg-info/SOURCES.txt @@ -0,0 +1,379 @@ +.coveragerc +.dockerignore +.stestr.conf +.zuul.yaml +AUTHORS +ChangeLog +HACKING.rst +LICENSE +Makefile +README.rst +REVIEWING.rst +alembic.ini +bindep.txt +entrypoint.sh +requirements-direct.txt +requirements-frozen.txt +requirements.txt +setup.cfg +setup.py +test-requirements.txt +tox.ini +.github/SECURITY.md +alembic/README +alembic/env.py +alembic/script.py.mako +alembic/versions/918bbfd28185_initial_deckhand_base.py +charts/deckhand/.helmignore +charts/deckhand/Chart.yaml +charts/deckhand/requirements.yaml +charts/deckhand/values.yaml +charts/deckhand/templates/configmap-bin.yaml +charts/deckhand/templates/configmap-etc.yaml +charts/deckhand/templates/deployment.yaml +charts/deckhand/templates/ingress-api.yaml +charts/deckhand/templates/job-db-init.yaml +charts/deckhand/templates/job-db-sync.yaml +charts/deckhand/templates/job-image-repo-sync.yaml +charts/deckhand/templates/job-ks-endpoints.yaml +charts/deckhand/templates/job-ks-service.yaml +charts/deckhand/templates/job-ks-user.yaml +charts/deckhand/templates/network_policy.yaml +charts/deckhand/templates/secret-db.yaml +charts/deckhand/templates/secret-ingress-tls.yaml +charts/deckhand/templates/secret-keystone-env.yaml +charts/deckhand/templates/service-ingress.yaml +charts/deckhand/templates/service.yaml +charts/deckhand/templates/bin/_db-sync.sh.tpl +charts/deckhand/templates/tests/test-deckhand-api.yaml +charts/deps/.gitkeep +deckhand/__init__.py +deckhand/cmd.py +deckhand/context.py +deckhand/errors.py +deckhand/factories.py +deckhand/policy.py +deckhand/service.py +deckhand/types.py +deckhand/Deckhand.egg-info/PKG-INFO +deckhand/Deckhand.egg-info/SOURCES.txt +deckhand/Deckhand.egg-info/dependency_links.txt +deckhand/Deckhand.egg-info/entry_points.txt +deckhand/Deckhand.egg-info/not-zip-safe +deckhand/Deckhand.egg-info/pbr.json +deckhand/Deckhand.egg-info/requires.txt +deckhand/Deckhand.egg-info/top_level.txt +deckhand/barbican/__init__.py +deckhand/barbican/cache.py +deckhand/barbican/client_wrapper.py +deckhand/barbican/driver.py +deckhand/client/__init__.py +deckhand/client/base.py +deckhand/client/buckets.py +deckhand/client/client.py +deckhand/client/exceptions.py +deckhand/client/revisions.py +deckhand/client/tags.py +deckhand/common/__init__.py +deckhand/common/document.py +deckhand/common/utils.py +deckhand/common/validation_message.py +deckhand/conf/__init__.py +deckhand/conf/config.py +deckhand/conf/opts.py +deckhand/control/__init__.py +deckhand/control/api.py +deckhand/control/base.py +deckhand/control/buckets.py +deckhand/control/common.py +deckhand/control/health.py +deckhand/control/middleware.py +deckhand/control/no_oauth_middleware.py +deckhand/control/revision_deepdiffing.py +deckhand/control/revision_diffing.py +deckhand/control/revision_documents.py +deckhand/control/revision_tags.py +deckhand/control/revisions.py +deckhand/control/rollback.py +deckhand/control/validations.py +deckhand/control/versions.py +deckhand/control/views/__init__.py +deckhand/control/views/document.py +deckhand/control/views/revision.py +deckhand/control/views/revision_tag.py +deckhand/control/views/validation.py +deckhand/db/__init__.py +deckhand/db/sqlalchemy/__init__.py +deckhand/db/sqlalchemy/api.py +deckhand/db/sqlalchemy/models.py +deckhand/engine/__init__.py +deckhand/engine/_replacement.py +deckhand/engine/cache.py +deckhand/engine/document_validation.py +deckhand/engine/layering.py +deckhand/engine/render.py +deckhand/engine/revision_diff.py +deckhand/engine/secrets_manager.py +deckhand/engine/utils.py +deckhand/engine/schemas/base_schema.yaml +deckhand/engine/schemas/certificate_authority_key_schema.yaml +deckhand/engine/schemas/certificate_authority_schema.yaml +deckhand/engine/schemas/certificate_key_schema.yaml +deckhand/engine/schemas/certificate_schema.yaml +deckhand/engine/schemas/layering_policy_schema.yaml +deckhand/engine/schemas/metadata_control.yaml +deckhand/engine/schemas/metadata_document.yaml +deckhand/engine/schemas/passphrase_schema.yaml +deckhand/engine/schemas/private_key_schema.yaml +deckhand/engine/schemas/public_key_schema.yaml +deckhand/engine/schemas/validation_policy_schema.yaml +deckhand/policies/__init__.py +deckhand/policies/base.py +deckhand/policies/document.py +deckhand/policies/revision.py +deckhand/policies/revision_tag.py +deckhand/policies/validation.py +deckhand/tests/__init__.py +deckhand/tests/deckhand.conf.test +deckhand/tests/test_utils.py +deckhand/tests/common/__init__.py +deckhand/tests/common/test_gabbi.py +deckhand/tests/functional/README.rst +deckhand/tests/functional/gabbits/document/document-crud-error-bucket-conflict.yaml +deckhand/tests/functional/gabbits/document/document-crud-success-multi-bucket.yaml +deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml +deckhand/tests/functional/gabbits/document/document-crud-success-single-bucket.yaml +deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml +deckhand/tests/functional/gabbits/layering/layering-multiple-bucket.yaml +deckhand/tests/functional/gabbits/layering/layering-policy-conflict.yaml +deckhand/tests/functional/gabbits/layering/layering-single-bucket.yaml +deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml +deckhand/tests/functional/gabbits/layering/layering-with-substitution-single-bucket.yaml +deckhand/tests/functional/gabbits/replacement/multi-layer-replacement.yaml +deckhand/tests/functional/gabbits/replacement/replacement.yaml +deckhand/tests/functional/gabbits/resources/chained-substitution.yaml +deckhand/tests/functional/gabbits/resources/deckhand-owned-sample.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-2-layers.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-3-layers.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-a.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-b.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-delete.yaml +deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-update.yaml +deckhand/tests/functional/gabbits/resources/design-doc-substitution-generic-sample.yaml +deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml +deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml +deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml +deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml +deckhand/tests/functional/gabbits/resources/layering-and-substitution-sample.yaml +deckhand/tests/functional/gabbits/resources/layering-needs-substitution-source.yaml +deckhand/tests/functional/gabbits/resources/passphrase.yaml +deckhand/tests/functional/gabbits/resources/replacement.yaml +deckhand/tests/functional/gabbits/resources/sample-doc.yaml +deckhand/tests/functional/gabbits/resources/sample-schema-v2.yaml +deckhand/tests/functional/gabbits/resources/sample-schema.yaml +deckhand/tests/functional/gabbits/resources/sample-tag-data.yaml +deckhand/tests/functional/gabbits/resources/substitution-results-in-none-bug.yaml +deckhand/tests/functional/gabbits/resources/ucp-sample-documents.yaml +deckhand/tests/functional/gabbits/resources/unusual-documents.yaml +deckhand/tests/functional/gabbits/revision/revision-crud-success-single-bucket.yaml +deckhand/tests/functional/gabbits/revision/revision-filters.yaml +deckhand/tests/functional/gabbits/revision-deepdiff/revision-deepdiff-success.yaml +deckhand/tests/functional/gabbits/revision-diff/revision-diff-rollback-null-success.yaml +deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml +deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters-negative.yaml +deckhand/tests/functional/gabbits/revision-documents/revision-documents-filters.yaml +deckhand/tests/functional/gabbits/revision-documents/revision-documents-multiple-filters.yaml +deckhand/tests/functional/gabbits/revision-rollback/rollback-success-single-bucket.yaml +deckhand/tests/functional/gabbits/revision-tag/revision-tag-success.yaml +deckhand/tests/functional/gabbits/substitution/substitution-chained-single-bucket.yaml +deckhand/tests/functional/gabbits/substitution/substitution-multiple-bucket.yaml +deckhand/tests/functional/gabbits/substitution/substitution-results-in-none-bug.yaml +deckhand/tests/functional/gabbits/substitution/substitution-single-bucket-generic.yaml +deckhand/tests/functional/gabbits/substitution/substitution-single-bucket.yaml +deckhand/tests/functional/gabbits/substitution/substitution-source-feeds-multi-dest.yaml +deckhand/tests/integration/README.rst +deckhand/tests/integration/__init__.py +deckhand/tests/integration/gabbits/document-crud-secret.yaml +deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml +deckhand/tests/integration/gabbits/document-render-secret.yaml +deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml +deckhand/tests/integration/gabbits/document-substitution-secret.yaml +deckhand/tests/integration/gabbits/revision-delete-all-secrets.yaml +deckhand/tests/unit/__init__.py +deckhand/tests/unit/base.py +deckhand/tests/unit/dh_fixtures.py +deckhand/tests/unit/fake_policy.py +deckhand/tests/unit/test_policy.py +deckhand/tests/unit/barbican/__init__.py +deckhand/tests/unit/barbican/test_cache.py +deckhand/tests/unit/common/__init__.py +deckhand/tests/unit/common/test_utils.py +deckhand/tests/unit/control/__init__.py +deckhand/tests/unit/control/base.py +deckhand/tests/unit/control/test_api_initialization.py +deckhand/tests/unit/control/test_base_controller.py +deckhand/tests/unit/control/test_buckets_controller.py +deckhand/tests/unit/control/test_errors.py +deckhand/tests/unit/control/test_health_controller.py +deckhand/tests/unit/control/test_middleware.py +deckhand/tests/unit/control/test_rendered_documents_controller.py +deckhand/tests/unit/control/test_revision_documents_controller.py +deckhand/tests/unit/control/test_revision_tags_controller.py +deckhand/tests/unit/control/test_revisions_controller.py +deckhand/tests/unit/control/test_revisions_deepdiff_controller.py +deckhand/tests/unit/control/test_revisions_diff_controller.py +deckhand/tests/unit/control/test_revisions_rollback_controller.py +deckhand/tests/unit/control/test_versions_controller.py +deckhand/tests/unit/db/__init__.py +deckhand/tests/unit/db/test_documents.py +deckhand/tests/unit/db/test_documents_negative.py +deckhand/tests/unit/db/test_layering_policies.py +deckhand/tests/unit/db/test_revision_documents.py +deckhand/tests/unit/db/test_revision_rollback.py +deckhand/tests/unit/db/test_revision_tags.py +deckhand/tests/unit/db/test_revision_tags_negative.py +deckhand/tests/unit/db/test_revisions.py +deckhand/tests/unit/engine/__init__.py +deckhand/tests/unit/engine/base.py +deckhand/tests/unit/engine/test_cache.py +deckhand/tests/unit/engine/test_document_layering.py +deckhand/tests/unit/engine/test_document_layering_and_replacement.py +deckhand/tests/unit/engine/test_document_layering_and_replacement_negative.py +deckhand/tests/unit/engine/test_document_layering_and_substitution.py +deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py +deckhand/tests/unit/engine/test_document_layering_negative.py +deckhand/tests/unit/engine/test_document_validation.py +deckhand/tests/unit/engine/test_document_validation_negative.py +deckhand/tests/unit/engine/test_revision_deepdiffing.py +deckhand/tests/unit/engine/test_revision_diffing.py +deckhand/tests/unit/engine/test_secrets_manager.py +deckhand/tests/unit/resources/sample_certificate.yaml +deckhand/tests/unit/resources/sample_certificate_authority.yaml +deckhand/tests/unit/resources/sample_certificate_authority_key.yaml +deckhand/tests/unit/resources/sample_certificate_key.yaml +deckhand/tests/unit/resources/sample_data_schema.yaml +deckhand/tests/unit/resources/sample_document.yaml +deckhand/tests/unit/resources/sample_document_simple.yaml +deckhand/tests/unit/resources/sample_layering_policy.yaml +deckhand/tests/unit/resources/sample_passphrase.yaml +deckhand/tests/unit/resources/sample_private_key.yaml +deckhand/tests/unit/resources/sample_public_key.yaml +deckhand/tests/unit/resources/sample_validation_policy.yaml +deckhand/tests/unit/views/__init__.py +deckhand/tests/unit/views/test_document_views.py +deckhand/tests/unit/views/test_revision_tag_views.py +deckhand/tests/unit/views/test_revision_views.py +doc/.gitignore +doc/requirements-docs.txt +doc/source/conf.py +doc/source/glossary.rst +doc/source/index.rst +doc/source/overview.rst +doc/source/releasenotes +doc/source/_static/.placeholder +doc/source/contributor/HACKING.rst +doc/source/contributor/REVIEWING.rst +doc/source/contributor/developer-overview.rst +doc/source/contributor/index.rst +doc/source/contributor/policy-enforcement.rst +doc/source/contributor/testing.rst +doc/source/diagrams/architecture-pegleg.uml +doc/source/diagrams/architecture.uml +doc/source/operators/api_client.rst +doc/source/operators/api_ref.rst +doc/source/operators/configuration.rst +doc/source/operators/exceptions.rst +doc/source/operators/index.rst +doc/source/operators/multi-distro-support.rst +doc/source/users/document-types.rst +doc/source/users/documents.rst +doc/source/users/encryption.rst +doc/source/users/getting-started.rst +doc/source/users/index.rst +doc/source/users/layering.rst +doc/source/users/rendering.rst +doc/source/users/replacement.rst +doc/source/users/revision-history.rst +doc/source/users/substitution.rst +doc/source/users/validation.rst +etc/deckhand/config-generator.conf +etc/deckhand/deckhand-paste.ini +etc/deckhand/deckhand.conf.sample +etc/deckhand/logging.conf.sample +etc/deckhand/noauth-paste.ini +etc/deckhand/policy-generator.conf +etc/deckhand/policy.yaml.sample +images/deckhand/Dockerfile.opensuse_15 +images/deckhand/Dockerfile.ubuntu_focal +releasenotes/notes/development-mode-51208c39e9eee34f.yaml +releasenotes/notes/fix-attribute-error-in-bucket-controller-b5c9e410823abd19.yaml +releasenotes/notes/fix-unique-layering-policy-bac2940c3deeba51.yaml +releasenotes/notes/list-validations-details-endpoint-c3f18963b1372e40.yaml +releasenotes/notes/oslo.policy-integration-f03ac6a7a2ccef5a.yaml +releasenotes/notes/revision-document-filtering-0e57274a4fc1bc07.yaml +releasenotes/notes/rm-pep8-ignores-974a114e488ff1ea.yaml +releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml +releasenotes/notes/validations-api-cf07c8b6b5040f67.yaml +releasenotes/source/conf.py +releasenotes/source/index.rst +releasenotes/source/unreleased.rst +releasenotes/source/_static/.placeholder +releasenotes/source/_templates/.placeholder +tools/build-docs.sh +tools/common-tests.sh +tools/functional-tests.sh +tools/helm_install.sh +tools/helm_tk.sh +tools/image_tags.py +tools/integration-tests.sh +tools/run_pifpaf.sh +tools/unit-tests.sh +tools/whitespace-linter.sh +tools/gate/playbooks/airskiff-deploy.yaml +tools/gate/playbooks/airskiff-reduce-site.yaml +tools/gate/playbooks/build-charts.yaml +tools/gate/playbooks/debug-report.yaml +tools/gate/playbooks/docker-image-build.yaml +tools/gate/playbooks/docker-image-tag.yaml +tools/gate/playbooks/git-config.yaml +tools/gate/playbooks/install-postgresql.yaml +tools/gate/playbooks/osh-infra-build.yaml +tools/gate/playbooks/osh-infra-collect-logs.yaml +tools/gate/playbooks/osh-infra-deploy-docker.yaml +tools/gate/playbooks/osh-infra-deploy-k8s.yaml +tools/gate/playbooks/osh-infra-upgrade-host.yaml +tools/gate/playbooks/roles +tools/gate/playbooks/run-functional-tests-docker.yaml +tools/gate/playbooks/run-functional-tests-uwsgi.yaml +tools/gate/playbooks/run-integration-tests-docker.yaml +tools/gate/playbooks/run-integration-tests-uwsgi.yaml +tools/gate/playbooks/vars.yaml +tools/gate/roles/build-charts/tasks/build-charts.yaml +tools/gate/roles/build-charts/tasks/main.yaml +tools/gate/roles/build-images/defaults/main.yaml +tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml +tools/gate/roles/build-images/tasks/main.yaml +tools/gate/roles/deploy-apparmor/tasks/main.yaml +tools/gate/roles/deploy-barbican/tasks/deploy-barbican.yaml +tools/gate/roles/deploy-barbican/tasks/main.yaml +tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml +tools/gate/roles/deploy-deckhand/tasks/main.yaml +tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml +tools/gate/roles/deploy-keystone-dependencies/tasks/main.yaml +tools/gate/roles/deploy-postgresql/tasks/deploy-postgresql.yaml +tools/gate/roles/deploy-postgresql/tasks/main.yaml +tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml +tools/gate/roles/disable-systemd-resolved/tasks/main.yaml +tools/gate/roles/generate-test-config/tasks/generate-test-config.yaml +tools/gate/roles/generate-test-config/tasks/generate-test-paste.yaml +tools/gate/roles/generate-test-config/tasks/main.yaml +tools/gate/roles/install-postgresql/tasks/install-postgresql.yaml +tools/gate/roles/install-postgresql/tasks/main.yaml +tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml +tools/gate/roles/install-test-requirements/tasks/main.yaml +tools/gate/roles/run-functional-tests/tasks/functional-tests.yaml +tools/gate/roles/run-functional-tests/tasks/main.yaml +tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml +tools/gate/roles/run-integration-tests/tasks/main.yaml +tools/gate/scripts/010-build-charts.sh +tools/gate/scripts/020-deploy-postgresql.sh \ No newline at end of file diff --git a/deckhand/Deckhand.egg-info/dependency_links.txt b/deckhand/Deckhand.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/deckhand/Deckhand.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/deckhand/Deckhand.egg-info/entry_points.txt b/deckhand/Deckhand.egg-info/entry_points.txt new file mode 100644 index 00000000..4c1b5448 --- /dev/null +++ b/deckhand/Deckhand.egg-info/entry_points.txt @@ -0,0 +1,6 @@ +[oslo.config.opts] +deckhand.conf = deckhand.conf.opts:list_opts + +[oslo.policy.policies] +deckhand = deckhand.policies:list_rules + diff --git a/deckhand/Deckhand.egg-info/not-zip-safe b/deckhand/Deckhand.egg-info/not-zip-safe new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/deckhand/Deckhand.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/deckhand/Deckhand.egg-info/pbr.json b/deckhand/Deckhand.egg-info/pbr.json new file mode 100644 index 00000000..e4d85d15 --- /dev/null +++ b/deckhand/Deckhand.egg-info/pbr.json @@ -0,0 +1 @@ +{"git_version": "3c748ee", "is_release": false} \ No newline at end of file diff --git a/deckhand/Deckhand.egg-info/requires.txt b/deckhand/Deckhand.egg-info/requires.txt new file mode 100644 index 00000000..809f2d1d --- /dev/null +++ b/deckhand/Deckhand.egg-info/requires.txt @@ -0,0 +1,73 @@ +alembic<=1.7.1 +amqp +Beaker +chardet<6.0.0,>=3.0.2 +charset_normalizer<4.0.0,>=2.0.0 +ConfigParser +coverage +cryptography==3.4.8 +deepdiff +docutils +falcon +hacking +importlib-metadata +Jinja2 +jsonpath_ng +jsonpath-rw-ext>=1.0.0 +jsonpickle +jsonschema +keystoneauth1 +keystonemiddleware +kombu +MarkupSafe<2.1.0 +networkx +nose +oslo.cache +oslo.concurrency +oslo.config +oslo.context +oslo.db +oslo.log +oslo.messaging +oslo.middleware +oslo.policy +oslo.serialization +oslo.utils +packaging +Paste +PasteDeploy +PasteScript +pbr +psycopg2-binary +pylibyaml +pymongo +pyparsing +pyproject-api +python-barbicanclient +python-dateutil +python-keystoneclient +python-memcached +python-subunit +PyYAML<=5.4.1 +reno +requests==2.23.0 +responses==0.12.1 +Routes +setuptools<=45.2.0 +six +Sphinx +sphinx-rtd-theme +SQLAlchemy<=1.3.20 +stevedore +testrepository +testresources +testscenarios +testtools +tiddlyweb +urllib3<=1.26,>=1.21.1 +uWSGI +virtualenv +Werkzeug +wheel +wsgi-intercept>=1.2.2 +yq diff --git a/deckhand/Deckhand.egg-info/top_level.txt b/deckhand/Deckhand.egg-info/top_level.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/deckhand/Deckhand.egg-info/top_level.txt @@ -0,0 +1 @@ + diff --git a/deckhand/barbican/client_wrapper.py b/deckhand/barbican/client_wrapper.py index ffa28c0c..acde76c6 100644 --- a/deckhand/barbican/client_wrapper.py +++ b/deckhand/barbican/client_wrapper.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import six from keystoneauth1 import loading from keystoneauth1 import session @@ -62,9 +63,9 @@ class BarbicanClientWrapper(object): self._cached_client = cli except barbican_exc.HTTPAuthError as e: - LOG.exception(str(e)) + LOG.exception(six.text_type(e)) raise errors.BarbicanClientException(code=e.status_code, - details=str(e)) + details=six.text_type(e)) return cli diff --git a/deckhand/barbican/driver.py b/deckhand/barbican/driver.py index b269ef73..91892e33 100644 --- a/deckhand/barbican/driver.py +++ b/deckhand/barbican/driver.py @@ -14,6 +14,7 @@ import ast import time +import six import barbicanclient from oslo_config import cfg @@ -89,11 +90,13 @@ class BarbicanDriver(object): 'payload': payload } - LOG.info('Storing encrypted data in Barbican for document [{}, {}]' - .format(secret_doc.schema, secret_doc.name)) + message = ('Storing encrypted data in Barbican for document' + ' [{}, {}]').format(secret_doc.schema, secret_doc.name) + LOG.info(message) for i in range(CONF.secret_create_attempts): - LOG.debug('Creating secret in Barbican, attempt {} of {}' - .format((i + 1), CONF.secret_create_attempts)) + message = ('Creating secret in Barbican, attempt {} of {}' + .format((i + 1), CONF.secret_create_attempts)) + LOG.debug(message) try: return self._do_create_secret(secret_args) except Exception: @@ -104,9 +107,10 @@ class BarbicanDriver(object): # This was not the last attempt, suppress the error and # try again after a brief sleep sleep_amount = (i + 1) - LOG.error('Caught an error while trying to create a ' - 'secret in Barbican, will try again in {} second' - .format(sleep_amount)) + message('Caught an error while trying to create a ' + 'secret in Barbican, will try again in {} second' + .format(sleep_amount)) + LOG.error(message) time.sleep(sleep_amount) def _do_create_secret(self, secret_args): @@ -122,18 +126,18 @@ class BarbicanDriver(object): **secret_args) except (barbicanclient.exceptions.HTTPAuthError, barbicanclient.exceptions.HTTPClientError) as e: - LOG.exception(str(e)) + LOG.exception(six.text_type(e)) raise errors.BarbicanClientException(code=e.status_code, - details=str(e)) + details=six.text_type(e)) except barbicanclient.exceptions.HTTPServerError as e: LOG.error('Caught %s error from Barbican, likely due to a ' 'configuration or deployment issue.', e.__class__.__name__) - raise errors.BarbicanServerException(details=str(e)) + raise errors.BarbicanServerException(details=six.text_type(e)) except barbicanclient.exceptions.PayloadException as e: LOG.error('Caught %s error from Barbican, because the secret ' 'payload type is unsupported.', e.__class__.__name__) - raise errors.BarbicanServerException(details=str(e)) + raise errors.BarbicanServerException(details=six.text_type(e)) def _base64_decode_payload(self, payload): # If the secret_type is 'opaque' then this implies the @@ -154,13 +158,13 @@ class BarbicanDriver(object): secret = cache.lookup_by_ref(self.barbicanclient, secret_ref) except (barbicanclient.exceptions.HTTPAuthError, barbicanclient.exceptions.HTTPClientError) as e: - LOG.exception(str(e)) + LOG.exception(six.text_type(e)) raise errors.BarbicanClientException(code=e.status_code, - details=str(e)) + details=six.text_type(e)) except (barbicanclient.exceptions.HTTPServerError, ValueError) as e: - LOG.exception(str(e)) - raise errors.BarbicanServerException(details=str(e)) + LOG.exception(six.text_type(e)) + raise errors.BarbicanServerException(details=six.text_type(e)) payload = secret.payload if secret.secret_type == 'opaque': # nosec @@ -181,8 +185,8 @@ class BarbicanDriver(object): return self.barbicanclient.call("secrets.delete", secret_ref) except (barbicanclient.exceptions.HTTPAuthError, barbicanclient.exceptions.HTTPServerError) as e: - LOG.exception(str(e)) - raise errors.BarbicanClientException(details=str(e)) + LOG.exception(six.text_type(e)) + raise errors.BarbicanClientException(details=six.text_type(e)) except barbicanclient.exceptions.HTTPClientError as e: if e.status_code == 404: LOG.warning('Could not delete secret %s because it was not ' diff --git a/deckhand/common/utils.py b/deckhand/common/utils.py index 9de3747c..4854584e 100644 --- a/deckhand/common/utils.py +++ b/deckhand/common/utils.py @@ -15,13 +15,13 @@ import ast import copy import re +import six import string from beaker.cache import CacheManager from beaker.util import parse_cache_config_options import jsonpath_ng from oslo_log import log as logging -import six from deckhand.common.document import DocumentDict as document_dict from deckhand.conf import config @@ -49,7 +49,7 @@ def to_camel_case(s): def to_snake_case(name): """Convert string to snake case.""" - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', str(name)) + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', six.text_type(name)) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() diff --git a/deckhand/conf/opts.py b/deckhand/conf/opts.py index d3c0227e..7492e465 100644 --- a/deckhand/conf/opts.py +++ b/deckhand/conf/opts.py @@ -11,12 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import collections import importlib import os import pkgutil +import collections + LIST_OPTS_FUNC_NAME = "list_opts" IGNORED_MODULES = ('opts', 'constants', 'utils') diff --git a/deckhand/control/base.py b/deckhand/control/base.py index b15a6908..7717046f 100644 --- a/deckhand/control/base.py +++ b/deckhand/control/base.py @@ -13,6 +13,7 @@ # limitations under the License. import falcon +import six from oslo_log import log as logging import yaml @@ -74,7 +75,7 @@ class BaseResource(object): raise falcon.HTTPBadRequest(description=error_msg) if expect_list: - bad_entries = [str(i + 1) for i, x in enumerate(data) + bad_entries = [six.text_type(i + 1) for i, x in enumerate(data) if not x or not isinstance(x, dict)] if bad_entries: error_msg = ( diff --git a/deckhand/control/buckets.py b/deckhand/control/buckets.py index 2d58bb62..17279ca3 100644 --- a/deckhand/control/buckets.py +++ b/deckhand/control/buckets.py @@ -51,7 +51,8 @@ class BucketsResource(api_base.BaseResource): doc_validator.validate_all() except deckhand_errors.InvalidDocumentFormat as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) for document in documents: if secrets_manager.SecretsManager.requires_encryption(document): @@ -81,6 +82,7 @@ class BucketsResource(api_base.BaseResource): except (deckhand_errors.DuplicateDocumentExists, deckhand_errors.SingletonDocumentConflict) as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) return created_documents diff --git a/deckhand/control/common.py b/deckhand/control/common.py index b33bc395..635b60cf 100644 --- a/deckhand/control/common.py +++ b/deckhand/control/common.py @@ -182,7 +182,8 @@ def get_rendered_docs(revision_id, cleartext_secrets=False, **filters): errors.UnknownSubstitutionError, errors.UnsupportedActionMethod) as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) except errors.EncryptionSourceNotFound as e: # This branch should be unreachable, but if an encryption source # wasn't found, then this indicates the controller fed bad data @@ -212,7 +213,8 @@ def _retrieve_documents_for_rendering(revision_id, **filters): layering_policy = db_api.document_get( **layering_policy_filters) except errors.DocumentNotFound as e: - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) else: documents.append(layering_policy) diff --git a/deckhand/control/revision_deepdiffing.py b/deckhand/control/revision_deepdiffing.py index 20fff464..6fe6c24e 100644 --- a/deckhand/control/revision_deepdiffing.py +++ b/deckhand/control/revision_deepdiffing.py @@ -13,6 +13,8 @@ # limitations under the License. import falcon +import six + from oslo_log import log as logging from oslo_utils import excutils @@ -32,19 +34,21 @@ class RevisionDeepDiffingResource(api_base.BaseResource): try: revision_id = int(revision_id) except ValueError: - raise errors.InvalidInputException(input_var=str(revision_id)) + raise errors.InvalidInputException(input_var=six.text_type( + revision_id)) try: comparison_revision_id = int(comparison_revision_id) except ValueError: raise errors.InvalidInputException( - input_var=str(comparison_revision_id)) + input_var=six.text_type(comparison_revision_id)) try: resp_body = revision_diff( revision_id, comparison_revision_id, deepdiff=True) except errors.RevisionNotFound as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp.status = falcon.HTTP_200 resp.body = resp_body diff --git a/deckhand/control/revision_diffing.py b/deckhand/control/revision_diffing.py index 6eeb6bb8..c8f8047d 100644 --- a/deckhand/control/revision_diffing.py +++ b/deckhand/control/revision_diffing.py @@ -13,6 +13,7 @@ # limitations under the License. import falcon +import six from oslo_log import log as logging from oslo_utils import excutils @@ -32,19 +33,21 @@ class RevisionDiffingResource(api_base.BaseResource): try: revision_id = int(revision_id) except ValueError: - raise errors.InvalidInputException(input_var=str(revision_id)) + raise errors.InvalidInputException(input_var=six.text_type( + revision_id)) try: comparison_revision_id = int(comparison_revision_id) except ValueError: raise errors.InvalidInputException( - input_var=str(comparison_revision_id)) + input_var=six.text_type(comparison_revision_id)) try: resp_body = revision_diff( revision_id, comparison_revision_id) except errors.RevisionNotFound as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp.status = falcon.HTTP_200 resp.body = resp_body diff --git a/deckhand/control/revision_tags.py b/deckhand/control/revision_tags.py index ab770579..9a1f837b 100644 --- a/deckhand/control/revision_tags.py +++ b/deckhand/control/revision_tags.py @@ -39,7 +39,8 @@ class RevisionTagsResource(api_base.BaseResource): errors.RevisionTagBadFormat, errors.errors.RevisionTagNotFound) as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp_body = revision_tag_view.ViewBuilder().show(resp_tag) resp.status = falcon.HTTP_201 @@ -60,7 +61,8 @@ class RevisionTagsResource(api_base.BaseResource): except (errors.RevisionNotFound, errors.RevisionTagNotFound) as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp_body = revision_tag_view.ViewBuilder().show(resp_tag) resp.status = falcon.HTTP_200 @@ -73,7 +75,8 @@ class RevisionTagsResource(api_base.BaseResource): resp_tags = db_api.revision_tag_get_all(revision_id) except errors.RevisionNotFound as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp_body = revision_tag_view.ViewBuilder().list(resp_tags) resp.status = falcon.HTTP_200 @@ -94,7 +97,8 @@ class RevisionTagsResource(api_base.BaseResource): except (errors.RevisionNotFound, errors.RevisionTagNotFound) as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp.status = falcon.HTTP_204 @@ -105,6 +109,7 @@ class RevisionTagsResource(api_base.BaseResource): db_api.revision_tag_delete_all(revision_id) except errors.RevisionNotFound as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp.status = falcon.HTTP_204 diff --git a/deckhand/control/revisions.py b/deckhand/control/revisions.py index fd66823a..cbb35a40 100644 --- a/deckhand/control/revisions.py +++ b/deckhand/control/revisions.py @@ -56,7 +56,8 @@ class RevisionsResource(api_base.BaseResource): revision = db_api.revision_get(revision_id) except errors.RevisionNotFound as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) revision_resp = self.view_builder.show(revision) resp.status = falcon.HTTP_200 diff --git a/deckhand/control/rollback.py b/deckhand/control/rollback.py index 2474e1ab..c60878b9 100644 --- a/deckhand/control/rollback.py +++ b/deckhand/control/rollback.py @@ -36,7 +36,8 @@ class RollbackResource(api_base.BaseResource): latest_revision = db_api.revision_get_latest() except errors.RevisionNotFound as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) for document in latest_revision['documents']: if document['metadata'].get('storagePolicy') == 'encrypted': @@ -49,7 +50,8 @@ class RollbackResource(api_base.BaseResource): revision_id, latest_revision) except errors.RevisionNotFound as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) revision_resp = self.view_builder.show(rollback_revision) resp.status = falcon.HTTP_201 diff --git a/deckhand/control/validations.py b/deckhand/control/validations.py index 80bfb5f3..30a8fefc 100644 --- a/deckhand/control/validations.py +++ b/deckhand/control/validations.py @@ -46,7 +46,8 @@ class ValidationsResource(api_base.BaseResource): revision_id, validation_name, validation_data) except errors.RevisionNotFound as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp.status = falcon.HTTP_201 resp.body = self.view_builder.show(resp_body) @@ -80,7 +81,8 @@ class ValidationsResource(api_base.BaseResource): except (errors.RevisionNotFound, errors.ValidationNotFound) as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp_body = self.view_builder.show_entry(entry) return resp_body @@ -93,7 +95,8 @@ class ValidationsResource(api_base.BaseResource): validation_name) except errors.RevisionNotFound as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp_body = self.view_builder.list_entries(entries) return resp_body @@ -104,7 +107,8 @@ class ValidationsResource(api_base.BaseResource): validations = db_api.validation_get_all(revision_id) except errors.RevisionNotFound as e: with excutils.save_and_reraise_exception(): - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) resp_body = self.view_builder.list(validations) return resp_body diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index b0e8d782..1bf26b3c 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -18,6 +18,7 @@ import copy import functools import hashlib import threading +import six from oslo_config import cfg from oslo_db import exception as db_exception @@ -866,7 +867,7 @@ def revision_rollback(revision_id, latest_revision, session=None): if revision_id == 0: # Delete all existing documents in all buckets all_buckets = bucket_get_all(deleted=False) - bucket_names = [str(b['name']) for b in all_buckets] + bucket_names = [six.text_type(b['name']) for b in all_buckets] revision = documents_delete_from_buckets_list(bucket_names, session=session) @@ -898,7 +899,7 @@ def revision_rollback(revision_id, latest_revision, session=None): all_buckets = bucket_get_all(deleted=False) for bucket in all_buckets: if bucket['id'] not in unique_buckets: - buckets_to_delete.append(str(bucket['name'])) + buckets_to_delete.append(six.text_type(bucket['name'])) # Create the new revision, if len(buckets_to_delete) > 0: diff --git a/deckhand/engine/document_validation.py b/deckhand/engine/document_validation.py index fb0f876b..9fbbf77b 100644 --- a/deckhand/engine/document_validation.py +++ b/deckhand/engine/document_validation.py @@ -241,13 +241,13 @@ class DataSchemaValidator(GenericValidator): :returns: Dictionary in the above format. """ - error_path = '.'.join([str(x) for x in error.path]) + error_path = '.'.join([six.text_type(x) for x in error.path]) if error_path: path_to_error_in_document = '.'.join([root_path, error_path]) else: path_to_error_in_document = root_path path_to_error_in_schema = '.' + '.'.join( - [str(x) for x in error.schema_path]) + [six.text_type(x) for x in error.schema_path]) parent_path_to_error_in_document = '.'.join( path_to_error_in_document.split('.')[:-1]) or '.' diff --git a/deckhand/engine/layering.py b/deckhand/engine/layering.py index 4160c8e3..605b0d2b 100644 --- a/deckhand/engine/layering.py +++ b/deckhand/engine/layering.py @@ -13,6 +13,7 @@ # limitations under the License. import collections + import copy import networkx diff --git a/deckhand/engine/render.py b/deckhand/engine/render.py index 8d714820..11c40b3a 100644 --- a/deckhand/engine/render.py +++ b/deckhand/engine/render.py @@ -82,7 +82,8 @@ def validate_render(revision_id, rendered_documents, validator): # rendering bug, so override the default code to 500. e.code = 500 LOG.error('Failed to post-validate rendered documents.') - LOG.exception(e.format_message()) + message = (e.format_message()) + LOG.exception(message) raise e error_list = [] diff --git a/deckhand/engine/revision_diff.py b/deckhand/engine/revision_diff.py index f8373c1c..357fea0a 100644 --- a/deckhand/engine/revision_diff.py +++ b/deckhand/engine/revision_diff.py @@ -15,6 +15,7 @@ from deepdiff import DeepDiff from deepdiff.helper import RemapDict import jsonpickle +import six from deckhand.control import common from deckhand.db.sqlalchemy import api as db_api @@ -253,17 +254,18 @@ def _diff_buckets(b1, b2): # deepdiff doesn't provide custom exceptions; # have to use Exception. except Exception as ex: - raise errors.DeepDiffException(details=str(ex)) + raise errors.DeepDiffException( + details=six.text_type(ex)) try: metadata_changed = jsonpickle.decode( DeepDiff(d['metadata'], b2_tmp[k]['metadata']).to_json()) except Exception as ex: - raise errors.DeepDiffException(details=str(ex)) + raise errors.DeepDiffException(details=six.text_type(ex)) change_details.update({ - str(k): {'data_changed': data_changed, - 'metadata_changed': metadata_changed}}) + six.text_type(k): {'data_changed': data_changed, + 'metadata_changed': metadata_changed}}) if change_count > 0: diff_result.update({'document_changed': { @@ -283,7 +285,7 @@ def _format_diff_result(dr): v = dict(v) dr.update({k: v}) if isinstance(v, type): - dr.update({k: str(v)}) + dr.update({k: six.text_type(v)}) if isinstance(v, dict): _format_diff_result(v) return dr diff --git a/deckhand/tests/test_utils.py b/deckhand/tests/test_utils.py index 72e5f101..5c049831 100644 --- a/deckhand/tests/test_utils.py +++ b/deckhand/tests/test_utils.py @@ -14,6 +14,7 @@ # under the License. import random +import six import string import uuid @@ -37,7 +38,7 @@ def rand_name(name='', prefix='deckhand'): (e.g. 'prefixfoo-namebar-154876201') :rtype: string """ - randbits = str(random.randint(1, 0x7fffffff)) + randbits = six.text_type(random.randint(1, 0x7fffffff)) rand_name = randbits if name: rand_name = name + '-' + rand_name diff --git a/deckhand/tests/unit/control/test_api_initialization.py b/deckhand/tests/unit/control/test_api_initialization.py index b5bd875f..07a94075 100644 --- a/deckhand/tests/unit/control/test_api_initialization.py +++ b/deckhand/tests/unit/control/test_api_initialization.py @@ -14,6 +14,7 @@ import inspect import os +import six from unittest import mock @@ -113,4 +114,4 @@ class TestApi(test_base.DeckhandTestCase): ], any_order=True) mock_db_api.setup_db.assert_called_once_with( - str(mock.sentinel.db_connection)) + six.text_type(mock.sentinel.db_connection)) diff --git a/deckhand/tests/unit/control/test_rendered_documents_controller.py b/deckhand/tests/unit/control/test_rendered_documents_controller.py index 1ab75a1b..50454a11 100644 --- a/deckhand/tests/unit/control/test_rendered_documents_controller.py +++ b/deckhand/tests/unit/control/test_rendered_documents_controller.py @@ -13,6 +13,7 @@ # limitations under the License. import re +import six import yaml from unittest import mock @@ -276,7 +277,7 @@ class TestRenderedDocumentsControllerRedaction(test_base.BaseControllerTest): headers={'Content-Type': 'application/x-yaml'}, params={ 'metadata.name': ['example-cert', 'deckhand-global'], - 'cleartext-secrets': str(cleartext_secrets) + 'cleartext-secrets': six.text_type(cleartext_secrets) }, params_csv=False) @@ -438,8 +439,8 @@ class TestRenderedDocumentsControllerEncrypted(test_base.BaseControllerTest): # Expect redacted data for all documents to be returned - # because the destination documents should receive redacted data. data = list(map(lambda x: x['data'], rendered_documents)) - self.assertTrue(redacted_password in data) - self.assertTrue(redacted_data in data) + self.assertIn(redacted_password, data) + self.assertIn(redacted_data, data) # Expect the substitutions to be redacted since both docs are # marked as encrypted diff --git a/deckhand/tests/unit/engine/test_document_layering_and_replacement_negative.py b/deckhand/tests/unit/engine/test_document_layering_and_replacement_negative.py index 2515f0c0..c302e1f9 100644 --- a/deckhand/tests/unit/engine/test_document_layering_and_replacement_negative.py +++ b/deckhand/tests/unit/engine/test_document_layering_and_replacement_negative.py @@ -39,8 +39,8 @@ class TestDocumentLayeringReplacementNegative( error_re = (r'.*Document replacement requires that both documents ' 'have the same `schema` and `metadata.name`.') - self.assertRaisesRegexp(errors.InvalidDocumentReplacement, error_re, - self._test_layering, documents) + self.assertRaisesRegex(errors.InvalidDocumentReplacement, error_re, + self._test_layering, documents) # Validate case where schemas mismatch. documents[1]['metadata']['schema'] = 'example/Kind/v1' @@ -49,8 +49,8 @@ class TestDocumentLayeringReplacementNegative( error_re = (r'Document replacement requires that both documents ' 'have the same `schema` and `metadata.name`.') - self.assertRaisesRegexp(errors.InvalidDocumentReplacement, error_re, - self._test_layering, documents) + self.assertRaisesRegex(errors.InvalidDocumentReplacement, error_re, + self._test_layering, documents) def test_non_replacement_same_name_and_schema_as_parent_raises_exc(self): """Validate that a non-replacement document (i.e. regular document @@ -70,8 +70,8 @@ class TestDocumentLayeringReplacementNegative( error_re = (r'.*Non-replacement documents cannot have the same ' '`schema` and `metadata.name`.*') - self.assertRaisesRegexp(errors.InvalidDocumentReplacement, error_re, - self._test_layering, documents) + self.assertRaisesRegex(errors.InvalidDocumentReplacement, error_re, + self._test_layering, documents) def test_replacement_without_parent_raises_exc(self): """Validate that attempting to do replacement without a parent document @@ -89,8 +89,8 @@ class TestDocumentLayeringReplacementNegative( error_re = (r'Document replacement requires that the document with ' '`replacement: true` have a parent.') - self.assertRaisesRegexp(errors.InvalidDocumentReplacement, error_re, - self._test_layering, documents) + self.assertRaisesRegex(errors.InvalidDocumentReplacement, error_re, + self._test_layering, documents) def test_replacement_with_parent_replace_true_raises_exc(self): """Validate that a parent document with replacement: true necessarily @@ -109,8 +109,8 @@ class TestDocumentLayeringReplacementNegative( error_re = (r'Document replacement requires that the document with ' '`replacement: true` have a parent.') - self.assertRaisesRegexp(errors.InvalidDocumentReplacement, error_re, - self._test_layering, documents) + self.assertRaisesRegex(errors.InvalidDocumentReplacement, error_re, + self._test_layering, documents) def test_replacement_that_is_replaced_raises_exc(self): """Validate that attempting to replace a replacement document raises an @@ -135,8 +135,8 @@ class TestDocumentLayeringReplacementNegative( error_re = (r'A replacement document cannot itself be replaced by ' 'another document.') - self.assertRaisesRegexp(errors.InvalidDocumentReplacement, error_re, - self._test_layering, documents) + self.assertRaisesRegex(errors.InvalidDocumentReplacement, error_re, + self._test_layering, documents) def test_replacement_true_with_parent_replacement_true_raises_exc(self): """Validate that when documents have the same ``metadata.name`` and @@ -157,5 +157,5 @@ class TestDocumentLayeringReplacementNegative( error_re = ( r'More than one document with the same name and schema was found, ' 'but none has `replacement: true`.*') - self.assertRaisesRegexp(errors.InvalidDocumentReplacement, error_re, - self._test_layering, documents) + self.assertRaisesRegex(errors.InvalidDocumentReplacement, error_re, + self._test_layering, documents) diff --git a/deckhand/tests/unit/engine/test_document_layering_negative.py b/deckhand/tests/unit/engine/test_document_layering_negative.py index 04f8ba21..ea16d602 100644 --- a/deckhand/tests/unit/engine/test_document_layering_negative.py +++ b/deckhand/tests/unit/engine/test_document_layering_negative.py @@ -231,7 +231,7 @@ class TestDocumentLayeringNegative( layering_policy['data']['layerOrder'] = [] error_re = "layer \'global\' .* was not found in layerOrder.*" - self.assertRaisesRegexp( + self.assertRaisesRegex( errors.InvalidDocumentLayer, error_re, self._test_layering, [layering_policy, document]) diff --git a/deckhand/tests/unit/engine/test_document_validation.py b/deckhand/tests/unit/engine/test_document_validation.py index 9a0a2c51..aab42647 100644 --- a/deckhand/tests/unit/engine/test_document_validation.py +++ b/deckhand/tests/unit/engine/test_document_validation.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import six + from unittest import mock from deckhand.common import utils @@ -95,8 +97,9 @@ class TestDocumentValidation(engine_test_base.TestDocumentValidationBase): self.assertEqual(1, len(validations[0]['errors'])) self.assertIn('Sanitized to avoid exposing secret.', - str(validations[0]['errors'][-1])) - self.assertNotIn('scary-secret.', str(validations[0]['errors'][-1])) + six.text_type(validations[0]['errors'][-1])) + self.assertNotIn('scary-secret.', + six.text_type(validations[0]['errors'][-1])) def test_validation_document_duplication(self): """Validate that duplicate document fails when duplicate passed in.""" diff --git a/deckhand/tests/unit/engine/test_document_validation_negative.py b/deckhand/tests/unit/engine/test_document_validation_negative.py index 37b774d2..c6729446 100644 --- a/deckhand/tests/unit/engine/test_document_validation_negative.py +++ b/deckhand/tests/unit/engine/test_document_validation_negative.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import six + from unittest import mock from deckhand.engine import document_validation @@ -165,7 +167,8 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): missing_property = parts[-1] error_re = r"%s is a required property" % missing_property - self.assertRegex(str(e.error_list).replace("\'", ""), error_re) + self.assertRegex(six.text_type(e.error_list).replace("\'", ""), + error_re) def test_layering_policy_missing_required_sections(self): properties_to_remove = tuple(self.BASIC_CONTROL_PROPERTIES) + ( @@ -185,7 +188,8 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): doc_validator = document_validation.DocumentValidation(payload) e = self.assertRaises(errors.InvalidDocumentFormat, doc_validator.validate_all) - self.assertRegex(str(e.error_list[0]).replace("\'", ""), error_re) + self.assertRegex(six.text_type(e.error_list[0]).replace("\'", ""), + error_re) def test_passphrase_missing_required_sections(self): document = self._read_data('sample_passphrase') @@ -241,7 +245,7 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): # Validate that broken built-in base schema raises RuntimeError. doc_validator = document_validation.DocumentValidation(document) doc_validator._validators[0].base_schema = 'fake' - with self.assertRaisesRegexp(RuntimeError, 'Unknown error'): + with self.assertRaisesRegex(RuntimeError, 'Unknown error'): doc_validator.validate_all() # Validate that broken data schema for ``DataSchemaValidator`` raises @@ -252,7 +256,7 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): data_schema['data'] = 'fake' doc_validator = document_validation.DocumentValidation( [document, data_schema], pre_validate=False) - with self.assertRaisesRegexp(RuntimeError, 'Unknown error'): + with self.assertRaisesRegex(RuntimeError, 'Unknown error'): doc_validator.validate_all() def test_parent_selector_but_no_actions_raises_validation_error(self): diff --git a/doc/requirements-docs.txt b/doc/requirements-docs.txt new file mode 100644 index 00000000..0a5deb72 --- /dev/null +++ b/doc/requirements-docs.txt @@ -0,0 +1,25 @@ +plantuml +sphinxcontrib-apidoc>=0.2.0 # BSD +sphinxcontrib-plantuml +Sphinx==6.1.3 +sphinx-rtd-theme==0.5.0 +reno==4.0.0 +pylibyaml==0.1.0 +oslo.config==9.1.1 +oslo.policy==4.1.1 + + +Beaker==1.12.1 +deepdiff==6.3.0 +falcon==3.1.1 +jsonpath-ng==1.5.3 +jsonschema==4.17.3 +keystoneauth1==5.1.2 +networkx==3.1 +Paste==3.5.2 +PasteDeploy==3.0.1 +python-barbicanclient==5.5.0 +oslo.db==10.0.0 +oslo.log==5.2.0 +Werkzeug==2.1.2 +jsonpickle==3.0.1 \ No newline at end of file diff --git a/doc/requirements.txt b/doc/requirements.txt deleted file mode 100644 index 916fb8ad..00000000 --- a/doc/requirements.txt +++ /dev/null @@ -1,30 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. -sphinx>=1.6.2 # BSD -sphinx_rtd_theme -reno>=2.5.0 # Apache-2.0 -pylibyaml==0.1.0 -plantuml -sphinxcontrib-apidoc>=0.2.0 # BSD -sphinxcontrib-plantuml - -# NOTE(felipemonteiro): Required by RTD to make oslo.policy and -# oslo.config sample generation work. -oslo.config>=7.0.0 # Apache-2.0 -oslo.policy>=1.33.1 # Apache-2.0 - -# NOTE(gorshunovr): from ../requirements.txt -beaker==1.10.0 -deepdiff==3.3.0 -falcon==1.4.1 -jsonpath-ng==1.4.3 -jsonschema>=3.0.1,<4 -keystoneauth1==3.11.1 -networkx==2.2 -Paste==3.0.1 -PasteDeploy==1.5.2 -python-barbicanclient==4.7.0 -oslo.db==4.41.1 -oslo.log==3.45.2 -Werkzeug==0.16.1 diff --git a/doc/source/users/getting-started.rst b/doc/source/users/getting-started.rst index 484cb241..f3951540 100644 --- a/doc/source/users/getting-started.rst +++ b/doc/source/users/getting-started.rst @@ -131,7 +131,7 @@ Install dependencies needed to spin up Deckhand via ``uwsgi``:: $ [sudo] pip install uwsgi $ virtualenv -p python3 /var/tmp/deckhand $ . /var/tmp/deckhand/bin/activate - $ pip install -r requirements.txt -r test-requirements.txt + $ pip install -r requirements-frozen.txt $ python setup.py install Afterward, create a sample configuration file automatically:: diff --git a/images/deckhand/Dockerfile.opensuse_15 b/images/deckhand/Dockerfile.opensuse_15 index 6ca805f4..d299df97 100644 --- a/images/deckhand/Dockerfile.opensuse_15 +++ b/images/deckhand/Dockerfile.opensuse_15 @@ -90,7 +90,7 @@ RUN chown -R deckhand: /home/deckhand \ # Set work directory and install dependencies WORKDIR /home/deckhand -RUN pip3 install -r requirements.txt +RUN pip3 install -r requirements-frozen.txt RUN python3 setup.py install # Set user to deckhand diff --git a/images/deckhand/Dockerfile.ubuntu_bionic b/images/deckhand/Dockerfile.ubuntu_bionic index 62b6a91c..863d0b54 100644 --- a/images/deckhand/Dockerfile.ubuntu_bionic +++ b/images/deckhand/Dockerfile.ubuntu_bionic @@ -97,4 +97,4 @@ USER deckhand # Execute entrypoint ENTRYPOINT ["/home/deckhand/entrypoint.sh"] -CMD ["server"] +CMD ["server"] \ No newline at end of file diff --git a/images/deckhand/Dockerfile.ubuntu_focal b/images/deckhand/Dockerfile.ubuntu_focal index 2fb34cdb..cae7b8b9 100644 --- a/images/deckhand/Dockerfile.ubuntu_focal +++ b/images/deckhand/Dockerfile.ubuntu_focal @@ -88,14 +88,15 @@ RUN chown -R deckhand: /home/deckhand \ # Set work directory and install dependencies WORKDIR /home/deckhand -RUN pip3 install --no-cache-dir -r requirements.txt +RUN pip3 install --no-cache-dir -r requirements-frozen.txt # Setting deckhand version for BPR ENV PBR_VERSION 1.0 COPY . /opt/deckhand -RUN pip3 install -e /opt/deckhand \ +# RUN python3 setup.py install --verbose +RUN pip3 install -e . --verbose --use-pep517 \ && echo "/opt/deckhand" \ > /usr/local/lib/python3.8/dist-packages/deckhand.pth diff --git a/requirements-direct.txt b/requirements-direct.txt new file mode 100644 index 00000000..ae8673ee --- /dev/null +++ b/requirements-direct.txt @@ -0,0 +1,77 @@ +alembic<=1.4.3 +amqp==2.6.1 +Beaker<=1.12.0 +chardet==3.0.4 +charset_normalizer >=2.0.0, < 4.0.0 +ConfigParser +coverage +cryptography==3.4.8 +deepdiff<=5.8.1 +docutils +falcon +hacking +importlib-metadata +Jinja2 +jsonpath_ng +jsonpath-rw-ext>=1.0.0 +jsonpickle==1.4.1 +jsonschema<=3.2.0 +keystoneauth1<=5.1.1 +keystonemiddleware +kombu<=4.6.11 +MarkupSafe<2.1.0, >=0.9.2 +networkx +nose +oslo.cache<=2.10.1 +oslo.concurrency +oslo.config<=8.7.1 +oslo.context<=4.1.0 +oslo.db<=10.0.0 +oslo.log<=4.6.0 +oslo.messaging<=12.13.0 +oslo.middleware<=4.4.0 +oslo.policy<=3.10.1 +oslo.serialization<=4.2.0 +oslo.utils<=4.12.3 +packaging==21.3 +Paste<=3.5.0 +PasteDeploy +PasteScript +pbr<=5.5.1 +psycopg2-binary +pycadf<=3.1.1 +pycodestyle<=2.6.0 +pyflakes<=2.2.0 +Pygments<=2.14.0 +pylibyaml==0.1.0 +pymongo +pyparsing<=2.4.7 +pyproject-api +python-barbicanclient<=5.2.0 +python-dateutil +python-keystoneclient +python-memcached +python-subunit<=1.4.0 +PyYAML<=5.4.1 +reno +requests==2.27.0 +Routes +setuptools<=45.2.0 +six +Sphinx +sphinx-rtd-theme==0.5.0 +SQLAlchemy<=1.3.20 +stevedore +testrepository +testresources +testscenarios +testtools<=2.5.0 +tiddlyweb +typing_extensions==4.5.0 +urllib3 >= 1.21.1, <= 1.25.11 +uWSGI==2.0.21 +virtualenv +Werkzeug +wheel +wsgi-intercept>=1.2.2 +yq diff --git a/requirements-frozen.txt b/requirements-frozen.txt new file mode 100644 index 00000000..3ca4eca0 --- /dev/null +++ b/requirements-frozen.txt @@ -0,0 +1,154 @@ +alabaster==0.7.13 +alembic==1.4.3 +amqp==2.6.1 +argcomplete==3.0.5 +attrs==23.1.0 +autopage==0.5.1 +Babel==2.12.1 +bcrypt==4.0.1 +Beaker==1.12.0 +cachetools==5.3.0 +certifi==2022.12.7 +cffi==1.15.1 +chardet==3.0.4 +charset-normalizer==2.0.12 +cliff==4.2.0 +cmd2==2.4.3 +configparser==5.3.0 +coverage==7.2.3 +cryptography==3.4.8 +debtcollector==2.5.0 +decorator==5.1.1 +deepdiff==5.8.1 +distlib==0.3.6 +dnspython==2.3.0 +docutils==0.19 +dogpile.cache==1.1.8 +dulwich==0.21.3 +eventlet==0.33.3 +extras==1.0.0 +falcon==3.1.1 +fasteners==0.18 +filelock==3.12.0 +fixtures==3.0.0 +flake8==3.8.4 +futurist==2.4.1 +greenlet==2.0.2 +hacking==4.1.0 +html5lib==0.9999999 +httpexceptor==1.4.0 +idna==3.4 +imagesize==1.4.1 +importlib-metadata==6.5.0 +iso8601==1.1.0 +Jinja2==3.1.2 +jsonpath-ng==1.5.3 +jsonpath-rw==1.4.0 +jsonpath-rw-ext==1.2.2 +jsonpickle==1.4.1 +jsonschema==3.2.0 +keystoneauth1==5.1.1 +keystonemiddleware==10.2.0 +kombu==4.6.11 +Mako==1.2.4 +MarkupSafe==2.0.1 +mccabe==0.6.1 +msgpack==1.0.5 +netaddr==0.8.0 +netifaces==0.11.0 +networkx==3.1 +nose==1.3.7 +ordered-set==4.1.0 +os-service-types==1.7.0 +oslo.cache==2.10.1 +oslo.concurrency==5.1.1 +oslo.config==8.7.1 +oslo.context==4.1.0 +oslo.db==10.0.0 +oslo.i18n==6.0.0 +oslo.log==4.6.0 +oslo.messaging==12.13.0 +oslo.metrics==0.6.0 +oslo.middleware==4.4.0 +oslo.policy==3.10.1 +oslo.serialization==4.2.0 +oslo.service==3.1.1 +oslo.utils==4.12.3 +packaging==21.3 +Paste==3.5.0 +PasteDeploy==3.0.1 +PasteScript==3.3.0 +pbr==5.5.1 +pip==23.1 +platformdirs==3.2.0 +ply==3.11 +prettytable==3.7.0 +prometheus-client==0.16.0 +psycopg2-binary==2.9.6 +pycadf==3.1.1 +pycodestyle==2.6.0 +pycparser==2.21 +pyflakes==2.2.0 +Pygments==2.14.0 +pylibyaml==0.1.0 +pymongo==4.3.3 +pyparsing==2.4.7 +pyperclip==1.8.2 +pyproject_api==1.5.0 +pyrsistent==0.19.3 +python-barbicanclient==5.2.0 +python-dateutil==2.8.2 +python-editor==1.0.4 +python-keystoneclient==5.1.0 +python-memcached==1.59 +python-mimeparse==1.6.0 +python-subunit==1.4.0 +pytz==2023.3 +PyYAML==5.4.1 +reno==4.0.0 +repoze.lru==0.7 +requests==2.27.0 +resolver==0.2.1 +rfc3986==2.0.0 +Routes==2.5.1 +selector==0.10.1 +setuptools==45.2.0 +simplejson==3.19.1 +six==1.16.0 +snowballstemmer==2.2.0 +Sphinx==6.1.3 +sphinx-rtd-theme==0.5.0 +sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +SQLAlchemy==1.3.20 +sqlalchemy-migrate==0.13.0 +sqlparse==0.4.4 +statsd==4.0.1 +stevedore==5.0.0 +Tempita==0.5.2 +testrepository==0.0.20 +testresources==2.0.1 +testscenarios==0.5.0 +testtools==2.5.0 +tiddlyweb==2.4.3 +tomli==2.0.1 +tomlkit==0.11.7 +typing_extensions==4.5.0 +urllib3==1.25.11 +uWSGI==2.0.21 +vine==1.3.0 +virtualenv==20.22.0 +wcwidth==0.2.6 +WebOb==1.8.7 +Werkzeug==2.1.2 +wheel==0.40.0 +wrapt==1.15.0 +wsgi-intercept==1.11.0 +xmltodict==0.13.0 +yappi==1.4.0 +yq==3.2.1 +zipp==3.15.0 diff --git a/requirements.txt b/requirements.txt index 582b8c26..5d4c01fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,55 +1,3 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - -hacking==3.0.1 -SQLAlchemy==1.4.23 -alembic==1.7.1 -amqp==5.0.8 -Beaker==1.12.0 -cryptography==3.4.8 -deepdiff==5.8.1 -falcon==3.1.1 -jsonpath-ng==1.5.3 -jsonschema==3.2.0 -keystoneauth1==5.1.1 -keystonemiddleware==10.2.0 -kombu==5.1.0 -networkx==2.6.2 -oslo.cache==2.8.2 -oslo.concurrency==4.4.1 -oslo.config==8.7.1 -oslo.context==5.0.0 -oslo.messaging==12.9.4 -oslo.db==11.0.0 -oslo.log==4.6.0 -oslo.middleware==4.4.0 -oslo.policy==4.0.0 -oslo.serialization==4.2.0 -oslo.utils==4.10.2 -pbr==5.6.0 -PasteDeploy==3.0.1 -Paste==3.5.0 -psycopg2-binary==2.9.5 -pylibyaml==0.1.0 -PyYAML==5.4.1 -python-dateutil==2.8.2 - -# TODO(alanmeadows) -# this must match the container service -# likely this should be imported from a -# container sidecar long-term -python-barbicanclient==5.2.0 -python-keystoneclient==3.22.0 -python-memcached==1.59 - -Routes==2.5.1 -six==1.16.0 -sphinx-rtd-theme==1.1.1 -stestr==3.2.0 -stevedore==4.1.1 -urllib3==1.26.6 -uWSGI==2.0.21 -# To support profiling in non-prod -Werkzeug==2.0.1 -jsonpickle \ No newline at end of file +# Warning: This file should be empty. +# Specify direct dependencies in requirements-direct.txt instead. +-r requirements-direct.txt diff --git a/setup.cfg b/setup.cfg index 4a9d9ea6..173ca167 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = Deckhand -version = 1.0 +version = 1.1 summary = Storage service for YAML-based configuration documents, which are managed through version control and automatically validated. description_file = README.rst author = The Airship Authors @@ -17,9 +17,16 @@ classifier = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.10 +[options] +include_package_data = True +python_requires = >=3.8 + + [files] packages = deckhand +data_files = + schemas = deckhand/engine/schemas/* [entry_points] oslo.config.opts = diff --git a/test-requirements.txt b/test-requirements.txt index 7f46f956..6a05704f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,29 +1,29 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. -amqp==5.0.8 -coverage==5.5 + +amqp +bandit==1.6.0 +chardet==3.0.4 fixtures==3.0.0 -python-subunit==1.4.0 +flake8==3.8.4 +gabbi==1.35.1 +jsonpickle==1.4.1 +openstacksdk==0.36.5 +openstacksdk==0.36.5 os-testr==2.0.1 +os-testr==2.0.1 +oslotest==4.5.0 +packaging==21.3 +pifpaf==3.1.5 +pylibyaml==0.1.0 +pyparsing==2.4.7 +pytest >= 3.0 +pytest-cov==4.0.0 +python-subunit==1.4.0 +requests==2.27.0 +six==1.16.0 +stestr==3.2.0 testrepository==0.0.20 testtools==2.5.0 -bandit==1.7.4 -# NOTE(felipemonteiro): Pin here because later versions require that -# content-type be present in empty responses. -gabbi==1.35.1 -pifpaf==3.1.5 -oslotest==4.5.0 -yq==3.1.0 -tox<=4 -pylibyaml==0.1.0 -six==1.16.0 -pyparsing==2.4.7 -openstacksdk==0.59.0 -stestr==3.2.0 -requests==2.28.2 -urllib3==1.26.6 -chardet==4.0.0 -pyproject-api==1.5.0 -packaging>=23 \ No newline at end of file +tox>=3.28.0, <4.0.0 +urllib3==1.25.11 +yq==3.2.1 diff --git a/tools/functional-tests.sh b/tools/functional-tests.sh index af026ead..162a4f8c 100755 --- a/tools/functional-tests.sh +++ b/tools/functional-tests.sh @@ -14,7 +14,7 @@ function cleanup_deckhand { set +e # Kill PostgreSQL if it is still running. - pifpaf_stop || deactive + # pifpaf_stop || deactive # Kill uwsgi service if it is still running. PID=$( sudo netstat -tulpn | grep ":9000" | head -n 1 | awk '{print $NF}' ) diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index 5bdb7b02..6634bd18 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -13,25 +13,13 @@ # limitations under the License. - hosts: primary + roles: + - clear-firewall + - bindep + - disable-systemd-resolved + - install-test-requirements + tasks: - - name: stop systemd-resolved service - systemd: - state: stopped - enabled: no - masked: yes - daemon_reload: yes - name: systemd-resolved - become: yes - - name: ensure pip is installed - apt: - name: "{{ item }}" - with_items: - - python-pip - - python3-pip - - python-setuptools - - python3-setuptools - when: ansible_os_family == 'Debian' - become: true - name: Clone Required Repositories shell: | export CLONE_DECKHAND={{ CLONE_DECKHAND }} @@ -59,9 +47,15 @@ chdir: "{{ zuul.project.src_dir }}" become: yes + - name: Setup clients + shell: | + ./tools/deployment/airskiff/developer/020-setup-client.sh + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + - name: Deploy Airship components using Armada shell: | - mkdir ~/.kube + mkdir -p ~/.kube cp -rp /home/zuul/.kube/config ~/.kube/config ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh args: diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml index 3a875353..0a4847f0 100644 --- a/tools/gate/playbooks/docker-image-build.yaml +++ b/tools/gate/playbooks/docker-image-build.yaml @@ -13,6 +13,12 @@ # limitations under the License. - hosts: primary + roles: + - bindep + - ensure-docker + - ensure-python + - ensure-pip + tasks: - name: Debug tag generation inputs block: @@ -40,23 +46,15 @@ debug: var: image_tags - - name: Install Docker (Debian) + - name: Install Docker python module for ansible docker login block: - - apt: - name: "{{ item }}" - with_items: - - docker.io - - python-pip - - python3-pip - - python-setuptools - - python3-setuptools - when: ansible_os_family == 'Debian' - pip: name: docker - version: 2.7.0 + version: 4.4.4 executable: pip3 become: True + - name: Make images when: not publish block: diff --git a/tools/gate/playbooks/osh-infra-build.yaml b/tools/gate/playbooks/osh-infra-build.yaml index d06296c1..2de61964 100644 --- a/tools/gate/playbooks/osh-infra-build.yaml +++ b/tools/gate/playbooks/osh-infra-build.yaml @@ -31,6 +31,8 @@ gather_facts: False become: yes roles: + - bindep - build-images tags: + - bindep - build-images diff --git a/tools/gate/playbooks/run-functional-tests-uwsgi.yaml b/tools/gate/playbooks/run-functional-tests-uwsgi.yaml index 434aadfe..7c9a1424 100644 --- a/tools/gate/playbooks/run-functional-tests-uwsgi.yaml +++ b/tools/gate/playbooks/run-functional-tests-uwsgi.yaml @@ -15,6 +15,10 @@ - hosts: all gather_facts: False roles: + - bindep + - ensure-docker + - ensure-python + - ensure-pip - disable-systemd-resolved - install-postgresql - install-test-requirements diff --git a/tools/gate/playbooks/run-integration-tests-docker.yaml b/tools/gate/playbooks/run-integration-tests-docker.yaml index 61e8c83c..8cb7b11d 100644 --- a/tools/gate/playbooks/run-integration-tests-docker.yaml +++ b/tools/gate/playbooks/run-integration-tests-docker.yaml @@ -16,6 +16,10 @@ vars_files: - vars.yaml roles: + - bindep + - ensure-docker + - ensure-python + - ensure-pip - install-test-requirements - build-images - deploy-keystone-dependencies diff --git a/tools/gate/playbooks/run-integration-tests-uwsgi.yaml b/tools/gate/playbooks/run-integration-tests-uwsgi.yaml index 0e1c0f58..74107e21 100644 --- a/tools/gate/playbooks/run-integration-tests-uwsgi.yaml +++ b/tools/gate/playbooks/run-integration-tests-uwsgi.yaml @@ -16,11 +16,18 @@ vars_files: - vars.yaml gather_facts: False + roles: + - clear-firewall + - bindep + - ensure-docker + - ensure-python + - ensure-pip - disable-systemd-resolved - install-postgresql - install-test-requirements - run-integration-tests + tags: - install-postgresql - run-integration-tests diff --git a/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml b/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml index 304894d9..195e264a 100644 --- a/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml +++ b/tools/gate/roles/deploy-deckhand/tasks/deploy-deckhand.yaml @@ -20,7 +20,6 @@ docker run \ --rm \ --net=host \ - --dns 10.96.0.10 \ -v "{{ deckhand_conf_dir | default('') }}":/etc/deckhand "{{ airship_deckhand_image_id.stdout }}" alembic upgrade head # Allow migrations to complete. @@ -35,7 +34,6 @@ --rm \ --net=host \ -p 9000:9000 \ - --dns 10.96.0.10 \ -v "{{ deckhand_conf_dir | default('') }}":/etc/deckhand "{{ airship_deckhand_image_id.stdout }}" server & # Give the server a chance to come up. Better to poll a health check. diff --git a/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml b/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml index 9e64855a..66b4333e 100644 --- a/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml +++ b/tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml @@ -16,14 +16,19 @@ shell: | set -xe; sudo modprobe br_netfilter + sudo sysctl net.bridge.bridge-nf-call-iptables=1 + sudo sysctl net.bridge.bridge-nf-call-ip6tables=1 - name: Install pip3 and gabbi shell: | set -xe; - apt-get install -y python3-pip \ - python-setuptools python3-setuptools + apt-get install -y \ + python3-pip \ + python3-setuptools \ + python3-dev pip3 install --upgrade pip pip3 install -r test-requirements.txt + pip3 install -r requirements-frozen.txt args: chdir: "{{ zuul.project.src_dir }}" become: yes diff --git a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml index 3d95ee16..035b75d9 100644 --- a/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml +++ b/tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. + + - name: Airship-Deckhand image build path shell: cd "{{ work_dir }}"; pwd register: airship_deckhand_path @@ -53,8 +55,8 @@ # NOTE(felipemonteiro): We don't use a venv because they don't play nicely # with OpenStack-Helm, which is used to orchestrate various OpenStack # services in the integration script called below. - sudo -H -E pip3 install -e . - sudo -H -E pip3 install -r requirements.txt -r test-requirements.txt + sudo -H -E pip3 install -e . --verbose --use-pep517 + sudo -E -H pip3 install -r requirements-frozen.txt pifpaf run postgresql -- ./tools/integration-tests.sh args: chdir: "{{ zuul.project.src_dir }}" diff --git a/tools/integration-tests.sh b/tools/integration-tests.sh index e2b7e755..4bd9ce60 100755 --- a/tools/integration-tests.sh +++ b/tools/integration-tests.sh @@ -31,7 +31,7 @@ function deploy_barbican { ./tools/deployment/common/wait-for-pods.sh openstack # Validate deployment info - helm status barbican + helm status barbican -n openstack } @@ -46,16 +46,23 @@ function deploy_osh_keystone_barbican { git clone https://git.openstack.org/openstack/openstack-helm.git ../openstack-helm fi + cd ${OSH_INFRA_PATH} + # git reset --hard ${BARBICAN_STABLE_COMMIT} + # Deploy required packages + ./tools/deployment/common/000-install-packages.sh + ./tools/deployment/common/001-setup-apparmor-profiles.sh + # cd ${OSH_PATH} # git reset --hard ${BARBICAN_STABLE_COMMIT} # Deploy required packages ./tools/deployment/common/install-packages.sh + # # Deploy Kubernetes sudo modprobe br_netfilter ./tools/deployment/common/deploy-k8s.sh cd ${CURRENT_DIR} - sudo -H -E pip install -r test-requirements.txt + sudo -H -E pip install -r requirements-frozen.txt # remove systemd-resolved local stub dns from resolv.conf sudo sed -i.bkp '/^nameserver.*127.0.0.1/d @@ -96,15 +103,16 @@ function deploy_deckhand { interfaces=("admin" "public" "internal") deckhand_endpoint="http://127.0.0.1:9000" - if [ -z "$( openstack service list --format value | grep deckhand )" ]; then - openstack service create --enable --name deckhand deckhand + if [ -z "$( openstack service list --format value 2>/dev/null | grep deckhand )" ]; then + openstack service create --enable --name deckhand deckhand 2>/dev/null fi for iface in ${interfaces[@]}; do - if [ -z "$( openstack endpoint list --format value | grep deckhand | grep $iface )" ]; then + if [ -z "$( openstack endpoint list --format value 2>/dev/null | grep deckhand | grep $iface )" ]; then openstack endpoint create --enable \ --region RegionOne \ - deckhand $iface $deckhand_endpoint/api/v1.0 + deckhand $iface $deckhand_endpoint/api/v1.0 \ + 2>/dev/null fi done @@ -126,8 +134,8 @@ function deploy_deckhand { # NOTE(fmontei): Generate an admin token instead of hacking a policy # file with no permissions to test authN as well as authZ. - export TEST_AUTH_TOKEN=$( openstack token issue --format value -c id ) - local test_barbican_url=$( openstack endpoint list --format value | grep barbican | grep public | awk '{print $7}' ) + export TEST_AUTH_TOKEN=$( openstack token issue --format value -c id 2>/dev/null ) + local test_barbican_url=$( openstack endpoint list --format value 2>/dev/null | grep barbican | grep public | awk '{print $7}' ) if [[ $test_barbican_url == */ ]]; then test_barbican_url=$( echo $test_barbican_url | sed 's/.$//' ) @@ -144,9 +152,9 @@ function run_tests { posargs=$@ if [ ${#posargs} -ge 1 ]; then - stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color $1 + stestr --test-path deckhand/tests/common/ run --verbose --serial --slowest --force-subunit-trace --color $1 else - stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color + stestr --test-path deckhand/tests/common/ run --verbose --serial --slowest --force-subunit-trace --color fi TEST_STATUS=$? diff --git a/tools/run_pifpaf.sh b/tools/run_pifpaf.sh index b9cfef20..428514b6 100755 --- a/tools/run_pifpaf.sh +++ b/tools/run_pifpaf.sh @@ -13,11 +13,11 @@ trap cleanup EXIT # Instantiate an ephemeral PostgreSQL DB and print out the `pifpaf` environment # variables for debugging purposes. set -ex -if [ -z $(which pg_config) ]; then +if [ -z "$(which pg_config)" ]; then sudo apt-get install libpq-dev postgresql -y fi -eval `pifpaf run postgresql` +eval "$(pifpaf run postgresql)" env | grep PIFPAF set +ex diff --git a/tools/whitespace-linter.sh b/tools/whitespace-linter.sh index a023399c..bfeeb560 100755 --- a/tools/whitespace-linter.sh +++ b/tools/whitespace-linter.sh @@ -3,12 +3,17 @@ set -xe RES=$(find . \ -not -path "*/\.*" \ + -not -path "*/venv/*" \ + -not -path "*/venv3/*" \ + -not -path "*/tmp.*" \ -not -path "*/*.egg-info/*" \ -not -path "*/releasenotes/build/*" \ -not -path "*/doc/build/*" \ -not -path "*/doc/source/images/*" \ + -not -path "*/keybd_*.png" \ -not -name "*.tgz" \ -not -name "*.html" \ + -not -name "favicon_32.png" \ -not -name "*.pyc" \ -type f -exec egrep -l " +$" {} \;) diff --git a/tox.ini b/tox.ini index 34d51ccf..f2591e08 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] -minversion = 3.4 +minversion = 3.28 # Flag indicating to perform the packaging operation or not. # Set it to true when using tox for an application, instead of a library. skipsdist = True -envlist = py38,py38-{postgresql},functional,cover,pep8,bandit,docs +envlist = py38,py38-{postgresql},functional-dev,cover,pep8,bandit,docs [testenv] # Install the current package in development mode with develop mode @@ -31,18 +31,39 @@ passenv = DECKHAND_IMAGE DECKHAND_TEST_URL DECKHAND_TEST_DIR -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +deps = + -r{toxinidir}/requirements-frozen.txt + -r{toxinidir}/test-requirements.txt commands = find . -type f -name "*.pyc" -delete rm -Rf .testrepository/times.dbm +exclude = venv + [testenv:venv] basepython=python3 commands = {posargs} +[testenv:freeze] +basepython=python3 +recreate = True +allowlist_externals= + rm + sh +deps= + -r{toxinidir}/requirements-direct.txt +commands= + rm -f requirements-frozen.txt + sh -c "pip freeze --all | grep -vE 'deckhand|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt" + + + [testenv:py38] +allowlist_externals = + stestr + find + rm commands = {[testenv]commands} stestr run {posargs} @@ -59,9 +80,12 @@ allowlist_externals = [testenv:functional] basepython=python3 +allowlist_externals = + find + stestr + pip3 + echo setenv = VIRTUAL_ENV={envdir} -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt commands = find . -type f -name "*.pyc" -delete stestr --test-path deckhand/tests/common/ run --serial --slowest --force-subunit-trace --color {posargs} @@ -71,15 +95,25 @@ basepython=python3 # Minimalistic functional test job for running Deckhand functional tests # via uwsgi. Uses pifpaf for DB instantiation. Useful for developers. # Requires PostgreSQL be installed on host. +allowlist_externals = + find + stestr + pifpaf + pip3 setenv = VIRTUAL_ENV={envdir} -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt commands = + pip3 install -e . --verbose --use-pep517 find . -type f -name "*.pyc" -delete pifpaf run postgresql -- {toxinidir}/tools/functional-tests.sh "{posargs}" [testenv:cover] -basepython = python3 +allowlist_externals = + find + stestr + coverage +deps = + -r{toxinidir}/requirements-frozen.txt + -r{toxinidir}/test-requirements.txt setenv = {[testenv]setenv} PYTHON=coverage run --source deckhand --parallel-mode commands = @@ -92,19 +126,21 @@ commands = coverage report [testenv:bandit] -basepython = python3 -commands = bandit -r deckhand -x deckhand/tests -n 5 +allowlist_externals = + bandit +commands = bandit -r deckhand --skip B311,B301,B106 -x deckhand/tests -n 5 [testenv:genconfig] -basepython = python3 commands = oslo-config-generator --config-file=etc/deckhand/config-generator.conf [testenv:genpolicy] -basepython = python3 commands = oslopolicy-sample-generator --config-file=etc/deckhand/policy-generator.conf [testenv:pep8] -basepython = python3 +allowlist_externals = + bandit + bash + flake8 deps = .[bandit] {[testenv]deps} @@ -114,7 +150,8 @@ commands = # Perform the actual pep8 flake8 {posargs} # Run security linter as part of the pep8 gate instead of using separate job. - bandit -r deckhand -x deckhand/tests -n 5 + bandit -r deckhand --skip B311,B301,B106 -x deckhand/tests -n 5 +exclude = venv [flake8] # [H106] Don't put vim configuration in source files. @@ -126,18 +163,16 @@ commands = enable-extensions = H106,H203,H204,H205,H210,H904 # [E731] Do not assign a lambda expression, use a def. This reduces readability in some cases. ignore = E731,F405,H405,W504,H306 -exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc,alembic/versions +exclude = .venv,venv,.git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc,alembic/versions,build/tmp.* [testenv:docs] -basepython = python3 deps = - -r{toxinidir}/doc/requirements.txt + -r{toxinidir}/doc/requirements-docs.txt commands = bash {toxinidir}/tools/build-docs.sh [testenv:releasenotes] -basepython = python3 -deps = -r{toxinidir}/doc/requirements.txt +deps = -r{toxinidir}/doc/requirements-docs.txt commands = rm -rf releasenotes/build sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html