diff --git a/docs/source/artifacts.rst b/docs/source/artifacts.rst new file mode 100644 index 00000000..7907e6cc --- /dev/null +++ b/docs/source/artifacts.rst @@ -0,0 +1,159 @@ +.. + Copyright 2018 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Definition Artifact Layout +========================== + +The definition artifacts are stored in the below directory structure. This +structure is used only to assist humans in maintaining the data. When the +documents are consumed by the UCP services, they are viewed as a flat set +of all documents.:: + + deployment_files/deployment_files + |- /global + | |- /common + | | |- {definition library} + | |- /v1.0 + | |- {definition library} + |- /type + | |- /production + | | |- /v1.0 + | | |- {definition library} + | |- /cicd + | | |- /v1.0 + | | |- {definition library} + | |- /labs + | |- /v1.0 + | |- {definition library} + |- /site + |- /{sitename} + |- site_definition.yaml + |- {definition library} + +The root-level listings of ``global``, ``type`` and ``site`` +are the layers as listed in the Deckhand +_LayeringPolicy http://deckhand.readthedocs.io/en/latest/layering.html +document. The process of choosing the definition libraries +to compose the actual design for a site is described below. + +site_definition.yaml +-------------------- + +The site_definition.yaml file is what selects the definition libraries +to use for a site. Additional metadata can be added to this file as needed +to meet requirements.:: + + --- + schema: pegleg/SiteDefinition/v1 + metadata: + layeringDefinition: + abstract: false + layer: 'site' + name: 'mtn13b.1' + schema: metadata/Document/v1 + storagePolicy: cleartext + data: + platform_name: 'integration' + revision: 'v1.0' + site_type: 'cicd' + +The ``revision`` field is used +to select the definition libraries in the ``global`` layer. This +layer will be composed of a union of documents in the ``common`` +definition library and the definition library +for the ``revision``. The ``revision`` field and +the ``site_type`` fields select the definition library from the +``type`` layer. And the ``site`` layer is defined by the single +defintion library under the sitename. + +Definition Library Layout +========================= + +The definition library layout is replicated in each location that the +site definition contains a set of documents.:: + + {library root} + |- /schemas + | |- /{namespace} + | |- /{kind} + | |- {version}.yaml + | + |- /profiles + | |- /hardware + | |- /host + | + |- /pki + | |- kubernetes-nodes.yaml + | + |- /secrets + | |- /certifcate-authorities + | |- /certificates + | |- /keypairs + | |- /passphrases + | + |- /software + | |- /charts + | | |- /{chart collection} + | | | |- dependencies.yaml + | | | |- /{chartgroup} + | | | |- chart-group.yaml + | | | |- {chart1}.yaml + | | | |- {chart2}.yaml + | | | + | | |- /{chart collection} + | | |- dependencies.yaml + | | |- /{chartgroup} + | | |- chart-group.yaml + | | |- {chart1}.yaml + | | |- {chart2}.yaml + | | + | |- /config + | | |- Docker.yaml + | | |- Kubelet.yaml + | | |- versions.yaml + | | + | |- /manifests + | |- bootstrap.yaml + | |- site.yaml + | + |- /networks + | |- /physical + | | |- sitewide.yaml + | | |- rack1.yaml + | | + | |- KubernetesNetwork.yaml + | |- common-addresses.yaml + | + |- /baremetal + |- rack1.yaml + |- rack2.yaml + + * Schemas - The schemas should all be sourced from the UCP + service repositories. Care should be taken that the schemas + included in the site definition are taken from the version of + the service being deployed in the site. + * Software + * /config/versions.yaml will contain a manifest of all the + chart, image and package versions. These should be substituted + into all other documents that define version information. + * dependencies.yaml - Contains Armada chart definitions that are + only utilized as dependencies for other charts (e.g. helm-toolkit) + * Chart collection - Loose organization of chart groups + such as 'kubernetes', 'ucp', 'osh' + * Physical networks and baremetal nodes can be split into files + in whatever way makes sense. The best practice here to define + them by racks is only a suggestion. + diff --git a/docs/source/authoring_strategy.rst b/docs/source/authoring_strategy.rst new file mode 100644 index 00000000..3e988e48 --- /dev/null +++ b/docs/source/authoring_strategy.rst @@ -0,0 +1,353 @@ +.. + Copyright 2018 AT&T Intellectual Property. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Document Fundamentals +===================== + +The definition of a site consists of a set of small YAML documents that +are managed by _Deckhand http://deckhand.readthedocs.io/en/latest/. Each +document is identified by a ``schema`` top-level key and the ``metadata.name`` +value that uniquely identifies a particular document of the type ``schema``. +Deckhand provides functionality allowing documents to be authored such that +data from multiple documents can be merged. + + * Abstact vs Concrete - Documents define a value in ``metadata.layeringDefinition.abstract`` to + determine if a document is abstract (a value of ``true``) or concrete (a value of ``false``). + When calling the ``/revisions/{id}/rendered-documents`` API, only concrete documents are returned. + * Layering - Document _layering http://deckhand.readthedocs.io/en/latest/layering.html is used + for whole documents that have known defaults but may need to be transformed in specific instances. + * Substitution - Data _substitution http://deckhand.readthedocs.io/en/latest/substitution.html is + used for extracting particular values from a document's data section (whole or in-part) and + inserting that data into a destination document (at the root of the data section or deeper + into a document). + +Shared Documents +================ + +Secrets +------- + +Several generic document +_types http://deckhand.readthedocs.io/en/latest/document_types.html#provided-utility-document-kinds +exist to support storing sensitive data encrypted. + +These must be utilized for all data considered sensitive. + +Global Catalogue Documents +-------------------------- + +Deckhand's layering functionality can be utilized in several ways, but AIC +site definitions will use a 'catalogue' approach. At the ``global`` layer +there will be several documents providing different configurations for an +object or service. Each of these will be abstract documents. They can be +incorporated into a particular site definition by creating a concrete +child document in the ``site`` layer that selects the correct ``global`` +parent. The child can then do further customization on the configuration if +needed. + +As a best practice, ``global`` level documents using the catalogue pattern +should utilize the layering labels ``component`` and ``configuration`` to +provide a consistent method for children documents select the correct parent. +The below example shows a set of documents for two configuration options for +OpenStack Keystone: one using local SQL-backed identity stores and one using +an LDAP backend. A site definition can then select and customize the appropriate +option. + +When using a catalogue document, it is important to review that document +to ensure you understand all the requirements for it. + + * Abstract documents are not required to be fully formed, so selecting + a catalogue document may require the child document to add data so + the document passes validation. In the below example, the child document + adds several required fields to the catalogue Chart: ``chart_name``, + ``release``, and ``namespace``. + * A catalogue document may define substitutions with the expectation + that the substitution source documents are defined at a lower layer. + In the example below, all of the required credentials in the chart + are defined as substitutions in the ``global`` catalogue document, + but the source documents for the substitutions are defined in the + ``site`` layer. + +This catalogue pattern can also be utilized for the ``type`` layer +if needed. + +Global Layer +------------ +.. highlight:: yaml + + --- + schema: armada/Chart/v1 + metadata: + schema: metadata/Document/v1 + name: ldap-backed-keystone + labels: + component: keystone + configuration: ldap-backed + layeringDefinition: + abstract: true + layer: global + storagePolicy: cleartext + substitutions: + - src: + schema: deckhand/Passphrase/v1 + name: keystone_admin_password + path: . + dest: + path: .values.endpoints.identity.auth.admin.password + - src: + schema: deckhand/Passphrase/v1 + name: mariadb_admin_password + path: . + dest: + path: .values.endpoints.oslo_db.auth.admin.password + - src: + schema: deckhand/Passphrase/v1 + name: mariadb_keystone_password + path: . + dest: + path: .values.endpoints.oslo_db.auth.user.password + - src: + schema: pegleg/SoftwareVersions/v1 + name: software-versions + path: .charts.ucp.keystone + dest: + path: .source + - src: + schema: pegleg/StringValue/v1 + name: ldap_userid + src: . + dest: + path: .values.conf.ks_domains.cicd.identity.ldap.user + pattern: '(^USERID)' + - src: + schema: deckhand/Passphrase/v1 + name: ldap_userid_password + path: . + dest: + path: .values.conf.ks_domain.cicd.identity.ldap.password + data: + install: + no_hooks: false + upgrade: + no_hooks: false + pre: + delete: + - type: job + labels: + job-name: keystone-db-sync + - type: job + labels: + job-name: keystone-db-init + post: + delete: [] + create: [] + values: + conf: + keystone: + identity: + driver: sql + default_domain_id: default + domain_specific_drivers_enabled: True + domain_configurations_from_database: True + domain_config_dir: /etc/keystonedomains + ks_domains: + cicd: + identity: + driver: ldap + ldap: + url: "ldap://your-ldap-server.example.com" + user: "USERID@example.com" + password: USERID_PASSWORD_REPLACEME + suffix: "dc=example,dc=com" + query_scope: sub + page_size: 1000 + user_tree_dn: "DC=example,DC=com" + user_objectclass: user + user_name_attribute: sAMAccountName + user_mail_attribute: mail + user_enabled_attribute: userAccountControl + user_enabled_mask: 2 + user_enabled_default: 512 + user_attribute_ignore: "default_project_id,tenants,projects,password" + replicas: 2 + labels: + node_selector_key: ucp-control-plane + node_selector_value: enabled + ... + --- + schema: armada/Chart/v1 + metadata: + schema: metadata/Document/v1 + name: sql-backed-keystone + labels: + component: keystone + configuration: sql-backed + layeringDefinition: + abstract: true + layer: global + substitutions: + - src: + schema: deckhand/Passphrase/v1 + name: keystone_admin_password + path: . + dest: + path: .values.endpoints.identity.auth.admin.password + - src: + schema: deckhand/Passphrase/v1 + name: mariadb_admin_password + path: . + dest: + path: .values.endpoints.oslo_db.auth.admin.password + - src: + schema: deckhand/Passphrase/v1 + name: mariadb_keystone_password + path: . + dest: + path: .values.endpoints.oslo_db.auth.user.password + - src: + schema: pegleg/SoftwareVersions/v1 + name: software-versions + path: .charts.ucp.keystone + dest: + path: .source + data: + timeout: 300 + install: + no_hooks: false + upgrade: + no_hooks: false + pre: + delete: + - name: keystone-bootstrap + type: job + labels: + application: keystone + component: bootstrap + - name: keystone-credential-setup + type: job + labels: + application: keystone + component: credential-setup + - name: keystone-db-init + type: job + labels: + application: keystone + component: db-init + - name: keystone-db-sync + type: job + labels: + application: keystone + component: db-sync + - name: keystone-fernet-setup + type: job + labels: + application: keystone + component: fernet-setup + values: {} + source: {} + ... + +Site Layer +---------- + +.. highlight:: yaml + + --- + schema: armada/Chart/v1 + metadata: + schema: metadata/Document/v1 + name: ucp-helm-toolkit + layeringDefinition: + abstract: false + layer: site + substitutions: + - src: + schema: pegleg/SoftwareVersions/v1 + name: software-versions + path: .charts.ucp.helm-toolkit + dest: + path: .source + data: + chart_name: ucp-helm-toolkit + release: ucp-helm-toolkit + namespace: ucp + timeout: 100 + values: {} + source: {} + dependencies: [] + ... + --- + schema: armada/Chart/v1 + metadata: + schema: metadata/Document/v1 + name: ucp-keystone + layeringDefinition: + abstract: false + layer: site + parentSelector: + component: keystone + configuration: ldap-backed + actions: + - method: merge + path: . + data: + chart_name: ucp-keystone + release: ucp-keystone + namespace: ucp + dependencies: + - ucp-helm-toolkit + ... + --- + schema: deckhand/Passphrase/v1 + metadata: + schema: metadata/Document/v1 + name: ldap_userid_password + storagePolicy: encrypted + data: a-secret-password + ... + --- + schema: deckhand/Passphrase/v1 + metadata: + schema: metadata/Document/v1 + name: keystone_admin_password + storagePolicy: encrypted + data: a-secret-password + ... + --- + schema: deckhand/Passphrase/v1 + metadata: + schema: metadata/Document/v1 + name: mariadb_admin_password + storagePolicy: encrypted + data: a-secret-password + ... + --- + schema: deckhand/Passphrase/v1 + metadata: + schema: metadata/Document/v1 + name: mariadb_keystone_password + storagePolicy: encrypted + data: a-secret-password + ... + --- + schema: pegleg/StringValue/v1 + metadata: + schema: metadata/Document/v1 + name: keystone_ldap_userid + storagePolicy: cleartext + data: myuser + ... + diff --git a/docs/source/conf.py b/docs/source/conf.py index 19eafe61..dcf30933 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,7 +51,7 @@ master_doc = 'index' # General information about the project. project = u'pegleg' -copyright = u'2017 AT&T Intellectual Property.' +copyright = u'2018 AT&T Intellectual Property.' author = u'pegleg Authors' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/source/index.rst b/docs/source/index.rst index b542728d..a7bcce6c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,5 +1,5 @@ .. - Copyright 2017 AT&T Intellectual Property. + Copyright 2018 AT&T Intellectual Property. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/tools/pegleg.sh b/tools/pegleg.sh old mode 100644 new mode 100755 index e910b399..df01c741 --- a/tools/pegleg.sh +++ b/tools/pegleg.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash -set -eu +set -e SCRIPT_DIR=$(realpath "$(dirname "${0}")") SOURCE_DIR=${SCRIPT_DIR}/pegleg -WORKSPACE=$(realpath "${SCRIPT_DIR}/..") +if [ -d "$PWD/global" ]; then + WORKSPACE="$PWD" +else + WORKSPACE=$(realpath "${SCRIPT_DIR}/..") +fi IMAGE_PEGLEG=${IMAGE_PEGLEG:-quay.io/attcomdev/pegleg:latest} diff --git a/tools/pegleg/pegleg/cli.py b/tools/pegleg/pegleg/cli.py index 73a7bfd4..6fe31833 100644 --- a/tools/pegleg/pegleg/cli.py +++ b/tools/pegleg/pegleg/cli.py @@ -103,37 +103,37 @@ def stub(): RELEASE_OPTION = click.option( '-r', - '--aic-revision', + '--revision', callback=_validate_revision_callback, required=True, - help='AIC revision to use (e.g. v4.0)') + help='Configuration revision to use (e.g. v1.0)') SITE_TYPE_OPTION = click.option( '-t', '--site-type', required=True, - help='Site type to use (e.g. "medium" or "large"') + help='Site type to use ("large", "medium", "cicd", "labs", etc.') @stub.command('global', help='Add global structure for a new revision') @RELEASE_OPTION -def global_(*, aic_revision): - engine.stub.global_(aic_revision) +def global_(*, revision): + engine.stub.global_(revision) @stub.command(help='Add a new site + revision') @click.argument('site_name') @RELEASE_OPTION @SITE_TYPE_OPTION -def site(*, aic_revision, site_type, site_name): - engine.stub.site(aic_revision, site_type, site_name) +def site(*, revision, site_type, site_name): + engine.stub.site(revision, site_type, site_name) @stub.command('site-type', help='Add a new site-type + revision') @RELEASE_OPTION @SITE_TYPE_OPTION -def site_type(*, aic_revision, site_type): - engine.stub.site_type(aic_revision, site_type) +def site_type(*, revision, site_type): + engine.stub.site_type(revision, site_type) @main.command(help='Sanity checks for repository content') diff --git a/tools/pegleg/pegleg/engine/site.py b/tools/pegleg/pegleg/engine/site.py index 383d893c..53c45dc8 100644 --- a/tools/pegleg/pegleg/engine/site.py +++ b/tools/pegleg/pegleg/engine/site.py @@ -27,7 +27,7 @@ def impacted(input_stream, output_stream): def list_(output_stream): - fieldnames = ['site_name', 'site_type', 'aic_revision'] + fieldnames = ['site_name', 'site_type', 'revision'] writer = csv.DictWriter( output_stream, fieldnames=fieldnames, delimiter=' ') for site_name in util.files.list_sites(): diff --git a/tools/pegleg/pegleg/engine/stub.py b/tools/pegleg/pegleg/engine/stub.py index b50f2504..4598de89 100644 --- a/tools/pegleg/pegleg/engine/stub.py +++ b/tools/pegleg/pegleg/engine/stub.py @@ -3,17 +3,17 @@ from pegleg.engine import util __all__ = ['global_', 'site', 'site_type'] -def global_(aic_revision): - util.files.create_global_directories(aic_revision) +def global_(revision): + util.files.create_global_directories(revision) -def site(aic_revision, site_type, site_name): +def site(revision, site_type, site_name): util.definition.create( - aic_revision=aic_revision, site_name=site_name, site_type=site_type) + revision=revision, site_name=site_name, site_type=site_type) params = util.definition.load_as_params(site_name) util.files.create_site_directories(**params) -def site_type(aic_revision, site_type): +def site_type(revision, site_type): util.files.create_site_type_directories( - aic_revision=aic_revision, site_type=site_type) + revision=revision, site_type=site_type) diff --git a/tools/pegleg/pegleg/engine/util/definition.py b/tools/pegleg/pegleg/engine/util/definition.py index a7e83ce3..967590ef 100644 --- a/tools/pegleg/pegleg/engine/util/definition.py +++ b/tools/pegleg/pegleg/engine/util/definition.py @@ -11,7 +11,7 @@ __all__ = [ ] -def create(*, site_name, site_type, aic_revision): +def create(*, site_name, site_type, revision): definition = { 'schema': 'pegleg/SiteDefinition/v1', 'metadata': { @@ -24,7 +24,7 @@ def create(*, site_name, site_type, aic_revision): }, }, 'data': { - 'aic_revision': aic_revision, + 'revision': revision, 'site_type': site_type, } } diff --git a/tools/pegleg/pegleg/engine/util/files.py b/tools/pegleg/pegleg/engine/util/files.py index 73793810..ea09c0b9 100644 --- a/tools/pegleg/pegleg/engine/util/files.py +++ b/tools/pegleg/pegleg/engine/util/files.py @@ -26,18 +26,18 @@ def all(): return search(DIR_DEPTHS.keys()) -def create_global_directories(aic_revision): +def create_global_directories(revision): _create_tree(_global_common_path()) - _create_tree(_global_revision_path(aic_revision)) + _create_tree(_global_revision_path(revision)) -def create_site_directories(*, site_name, aic_revision, **_kwargs): +def create_site_directories(*, site_name, revision, **_kwargs): _create_tree(_site_path(site_name)) -def create_site_type_directories(*, aic_revision, site_type): +def create_site_type_directories(*, revision, site_type): _create_tree(_site_type_common_path(site_type)) - _create_tree(_site_type_revision_path(site_type, aic_revision)) + _create_tree(_site_type_revision_path(site_type, revision)) FULL_STRUCTURE = { @@ -82,12 +82,12 @@ def _create_tree(root_path, *, tree=FULL_STRUCTURE): _create_tree(path, tree=data) -def directories_for(*, site_name, aic_revision, site_type): +def directories_for(*, site_name, revision, site_type): return [ _global_common_path(), - _global_revision_path(aic_revision), + _global_revision_path(revision), _site_type_common_path(site_type), - _site_type_revision_path(site_type, aic_revision), + _site_type_revision_path(site_type, revision), _site_path(site_name), ] @@ -96,16 +96,16 @@ def _global_common_path(): return 'global/common' -def _global_revision_path(aic_revision): - return 'global/%s' % aic_revision +def _global_revision_path(revision): + return 'global/%s' % revision def _site_type_common_path(site_type): return 'type/%s/common' % site_type -def _site_type_revision_path(site_type, aic_revision): - return 'type/%s/%s' % (site_type, aic_revision) +def _site_type_revision_path(site_type, revision): + return 'type/%s/%s' % (site_type, revision) def _site_path(site_name): @@ -137,7 +137,7 @@ def slurp(path): if not os.path.exists(path): raise click.ClickException( '%s not found. pegleg must be run from ' - 'the root of an AIC cLCP configuration repostiory.' % path) + 'the root of a configuration repostiory.' % path) with open(path) as f: try: