diff --git a/charts/promenade/templates/configmap-etc.yaml b/charts/promenade/templates/configmap-etc.yaml index 13d78be0..27411764 100644 --- a/charts/promenade/templates/configmap-etc.yaml +++ b/charts/promenade/templates/configmap-etc.yaml @@ -17,6 +17,34 @@ limitations under the License. {{- if .Values.manifests.configmap_etc }} {{- $envAll := . }} + +{{- if empty .Values.conf.promenade.keystone_authtoken.auth_uri -}} +{{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.promenade.keystone_authtoken "auth_uri" | quote | trunc 0 -}} +{{- end -}} + +# FIXME(sh8121att) fix for broken keystonemiddleware oslo config gen in newton - will remove in future +{{- if empty .Values.conf.promenade.keystone_authtoken.auth_url -}} +{{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup"| set .Values.conf.promenade.keystone_authtoken "auth_url" | quote | trunc 0 -}} +{{- end -}} + +{{- $userIdentity := .Values.endpoints.identity.auth.user -}} + +{{- if empty .Values.conf.promenade.keystone_authtoken.project_name -}} +{{- set .Values.conf.promenade.keystone_authtoken "project_name" $userIdentity.project_name | quote | trunc 0 -}} +{{- end -}} +{{- if empty .Values.conf.promenade.keystone_authtoken.project_domain_name -}} +{{- set .Values.conf.promenade.keystone_authtoken "project_domain_name" $userIdentity.project_domain_name | quote | trunc 0 -}} +{{- end -}} +{{- if empty .Values.conf.promenade.keystone_authtoken.user_domain_name -}} +{{- set .Values.conf.promenade.keystone_authtoken "user_domain_name" $userIdentity.user_domain_name | quote | trunc 0 -}} +{{- end -}} +{{- if empty .Values.conf.promenade.keystone_authtoken.username -}} +{{- set .Values.conf.promenade.keystone_authtoken "username" $userIdentity.username | quote | trunc 0 -}} +{{- end -}} +{{- if empty .Values.conf.promenade.keystone_authtoken.password -}} +{{- set .Values.conf.promenade.keystone_authtoken "password" $userIdentity.password | quote | trunc 0 -}} +{{- end -}} + --- apiVersion: v1 kind: ConfigMap @@ -25,4 +53,6 @@ metadata: data: api-paste.ini: |+ {{ include "helm-toolkit.utils.to_ini" .Values.conf.paste | indent 4 }} + promenade.conf: |+ +{{ include "helm-toolkit.utils.to_ini" .Values.conf.promenade | indent 4 }} {{- end }} diff --git a/charts/promenade/templates/deployment-api.yaml b/charts/promenade/templates/deployment-api.yaml index c5d6a750..a1b2e63d 100644 --- a/charts/promenade/templates/deployment-api.yaml +++ b/charts/promenade/templates/deployment-api.yaml @@ -61,6 +61,10 @@ spec: mountPath: /etc/promenade/api-paste.ini subPath: api-paste.ini readOnly: true + - name: promenade-etc + mountPath: /etc/promenade/promenade.conf + subPath: promenade.conf + readOnly: true volumes: - name: promenade-etc configMap: diff --git a/charts/promenade/values.yaml b/charts/promenade/values.yaml index 653c8c5f..7764ff74 100644 --- a/charts/promenade/values.yaml +++ b/charts/promenade/values.yaml @@ -13,6 +13,12 @@ # limitations under the License. conf: + promenade: + keystone_authtoken: + delay_auth_decision: true + auth_type: password + auth_section: keystone_authtoken + paste: pipeline:main: pipeline: authtoken promenade-api @@ -22,6 +28,7 @@ conf: forged_roles: admin paste.filter_factory: promenade.control.middleware:noauth_filter_factory app:promenade-api: + disable: "" paste.app_factory: promenade.promenade:paste_start_promenade images: diff --git a/examples/basic/LayeringPolicy.yaml b/examples/basic/LayeringPolicy.yaml new file mode 100644 index 00000000..46ae0c58 --- /dev/null +++ b/examples/basic/LayeringPolicy.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/LayeringPolicy/v1 +metadata: + schema: metadata/Control/v1 + name: layering-policy +data: + layerOrder: + - global + - type + - site +... diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index c76ea09d..d644a3be 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: release_prefix: ucp chart_groups: @@ -22,6 +23,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Kubernetes proxy sequenced: true @@ -35,6 +37,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Container networking via Calico sequenced: true @@ -49,6 +52,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Cluster DNS chart_group: @@ -61,6 +65,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Kubernetes components chart_group: @@ -76,6 +81,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: UCP platform components chart_group: @@ -85,6 +91,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: helm-toolkit + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: helm-toolkit release: helm-toolkit @@ -551,6 +561,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -651,6 +662,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -722,6 +734,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -783,6 +796,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -1022,6 +1036,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: chart_name: promenade release: promenade diff --git a/examples/complete/Docker.yaml b/examples/complete/Docker.yaml index 9b303fad..5d261396 100644 --- a/examples/complete/Docker.yaml +++ b/examples/complete/Docker.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: config: insecure-registries: diff --git a/examples/complete/Genesis.yaml b/examples/complete/Genesis.yaml index 70899c1a..8a5f5a1c 100644 --- a/examples/complete/Genesis.yaml +++ b/examples/complete/Genesis.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: hostname: n0 ip: 192.168.77.10 diff --git a/examples/complete/HostSystem.yaml b/examples/complete/HostSystem.yaml index 6f4eebd7..d3f7c2b0 100644 --- a/examples/complete/HostSystem.yaml +++ b/examples/complete/HostSystem.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: files: - path: /opt/kubernetes/bin/kubelet diff --git a/examples/complete/Kubelet.yaml b/examples/complete/Kubelet.yaml index 50016033..858cd4ff 100644 --- a/examples/complete/Kubelet.yaml +++ b/examples/complete/Kubelet.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: arguments: - --cni-bin-dir=/opt/cni/bin diff --git a/examples/complete/KubernetesNetwork.yaml b/examples/complete/KubernetesNetwork.yaml index b5755010..9c3d0373 100644 --- a/examples/complete/KubernetesNetwork.yaml +++ b/examples/complete/KubernetesNetwork.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: dns: cluster_domain: cluster.local diff --git a/examples/complete/LayeringPolicy.yaml b/examples/complete/LayeringPolicy.yaml new file mode 100644 index 00000000..46ae0c58 --- /dev/null +++ b/examples/complete/LayeringPolicy.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/LayeringPolicy/v1 +metadata: + schema: metadata/Control/v1 + name: layering-policy +data: + layerOrder: + - global + - type + - site +... diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml index 0654d6a8..de8b9600 100644 --- a/examples/complete/armada-resources.yaml +++ b/examples/complete/armada-resources.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: release_prefix: ucp chart_groups: @@ -24,6 +25,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Kubernetes proxy sequenced: true @@ -37,6 +39,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Container networking via Calico sequenced: true @@ -51,6 +54,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Cluster DNS chart_group: @@ -63,6 +67,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Ceph Storage sequenced: true @@ -77,6 +82,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Kubernetes components chart_group: @@ -92,6 +98,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: UCP Infrastructure sequenced: false @@ -101,7 +108,6 @@ data: - ucp-keystone - maas-postgresql - maas - - ucp-etcd-rabbitmq - ucp-rabbitmq - ucp-barbican - ingress @@ -114,6 +120,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: UCP platform components chart_group: @@ -127,6 +134,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: helm-toolkit + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: helm-toolkit release: helm-toolkit @@ -151,6 +162,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -207,6 +219,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -451,6 +464,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -514,6 +528,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -593,6 +608,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -693,6 +709,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -764,6 +781,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -825,6 +843,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -1061,6 +1080,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: ceph + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: ceph release: ceph @@ -1069,17 +1092,11 @@ data: wait: timeout: 3600 install: - no_hooks: false + no_hooks: true upgrade: no_hooks: false pre: delete: - - name: ceph-bootstrap - type: job - labels: - - application: ceph - - component: bootstrap - - release_group: armada-ucp - name: ceph-mds-keyring-generator type: job labels: @@ -1133,6 +1150,7 @@ data: storage_secrets: true ceph: true rbd_provisioner: true + cephfs_provisioner: true client_secrets: false rgw_keystone_user_and_endpoints: false bootstrap: @@ -1148,6 +1166,7 @@ data: ceph_config_helper: docker.io/port/ceph-config-helper:v1.7.5 ceph_daemon: docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04 ceph_rbd_provisioner: quay.io/external_storage/rbd-provisioner:v0.1.1 + ceph_cephfs_provisioner: quay.io/external_storage/cephfs-provisioner:v0.1.1 dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 ks_endpoints: docker.io/kolla/ubuntu-source-heat-engine:3.0.3 ks_service: docker.io/kolla/ubuntu-source-heat-engine:3.0.3 @@ -1164,6 +1183,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: ucp-ceph-config + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: ucp-ceph-config release: ucp-ceph-config @@ -1213,6 +1236,7 @@ data: deployment: storage_secrets: false ceph: false + cephfs_provisioner: false rbd_provisioner: false client_secrets: true rgw_keystone_user_and_endpoints: false @@ -1228,6 +1252,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: ucp-mariadb + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: ucp-mariadb release: ucp-mariadb @@ -1261,6 +1289,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: ucp-memcached + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: ucp-memcached release: ucp-memcached @@ -1288,6 +1320,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: ucp-keystone + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: ucp-keystone release: keystone @@ -1345,6 +1381,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: maas-postgresql + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: maas-postgresql release: maas-postgresql @@ -1381,6 +1421,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: maas + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: maas release: maas @@ -1392,15 +1436,13 @@ data: values: images: tags: + bootstrap: sthussey/maas-region-controller:2.3 db_init: docker.io/postgres:9.5 - db_sync: quay.io/attcomdev/maas-region:latest - dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 - maas_cache: docker.io/sthussey/maas-cache:cachetest - maas_rack: quay.io/attcomdev/maas-rack:latest - maas_region: quay.io/attcomdev/maas-region:latest - bootstrap: quay.io/attcomdev/maas-region:latest - export_api_key: quay.io/attcomdev/maas-region:latest + db_sync: sthussey/maas-region-controller:2.3 dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 + export_api_key: sthussey/maas-region-controller:2.3 + maas_rack: sthussey/maas-rack-controller:2.3 + maas_region: sthussey/maas-region-controller:2.3 labels: rack: node_selector_key: ucp-control-plane @@ -1419,7 +1461,7 @@ data: port: 31900 conf: drydock: - bootaction_url: http://${DRYDOCK_NODE_IP}:${DRYDOCK_NODE_PORT}/api/v1.0/bootactions/nodes/ + bootaction_url: http://192.168.77.10:31000/api/v1.0/bootactions/nodes/ maas: credentials: secret: @@ -1446,44 +1488,13 @@ data: - helm-toolkit --- schema: armada/Chart/v1 -metadata: - schema: metadata/Document/v1 - name: ucp-etcd-rabbitmq -data: - chart_name: ucp-etcd-rabbitmq - release: etcd-rabbitmq - namespace: ucp - install: - no_hooks: false - upgrade: - no_hooks: false - pre: - delete: [] - post: - delete: [] - create: [] - values: - pod: - replicas: - etcd: 1 - labels: - node_selector_key: ucp-control-plane - node_selector_value: enabled - images: - tags: - dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 - etcd: gcr.io/google_containers/etcd-amd64:2.2.5 - source: - type: git - location: https://git.openstack.org/openstack/openstack-helm - subpath: etcd - dependencies: - - helm-toolkit ---- -schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: ucp-rabbitmq + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: ucp-rabbitmq release: rabbitmq @@ -1500,7 +1511,7 @@ data: values: images: tags: - rabbitmq: quay.io/attcomdev/fuel-mcp-rabbitmq:ocata-unstable + rabbitmq: docker.io/rabbitmq:3.7 dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 pod: replicas: @@ -1519,6 +1530,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: ucp-barbican + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: ucp-barbican release: barbican @@ -1562,6 +1577,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: ingress + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: ingress release: ingress @@ -1580,10 +1599,10 @@ data: images: tags: entrypoint: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 - dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 - # https://github.com/kubernetes/ingress/blob/master/controllers/nginx/Changelog.md - ingress: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.8 + # https://github.com/kubernetes/ingress-nginx/blob/09524cd3363693463da5bf4a9bb3900da435ad05/Changelog.md#090 + ingress: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0 error_pages: gcr.io/google_containers/defaultbackend:1.0 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 source: type: git location: https://github.com/openstack/openstack-helm @@ -1596,6 +1615,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: tiller + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: tiller release: tiller @@ -1623,6 +1646,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: deckhand + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: deckhand release: deckhand @@ -1664,6 +1691,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: drydock + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: drydock release: drydock @@ -1716,6 +1747,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: chart_name: promenade release: promenade @@ -1724,17 +1756,6 @@ data: wait: timeout: 600 values: - conf: - paste: - filter:authtoken: - paste.filter_factory: 'keystonemiddleware.auth_token:filter_factory' - admin_password: password - admin_tenant_name: service - admin_user: promenade - delay_auth_decision: true - identity_uri: 'http://keystone-api.ucp.svc.cluster.local/' - service_token_roles_required: true - images: tags: dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 @@ -1755,6 +1776,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: armada + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: armada release: armada @@ -1794,6 +1819,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: shipyard + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: shipyard release: shipyard diff --git a/promenade/config.py b/promenade/config.py index 29680c75..25aee841 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -1,8 +1,8 @@ from . import exceptions, logging, validation +from .design_ref import get_documents import copy import jinja2 import jsonpath_ng -import requests import yaml __all__ = ['Configuration'] @@ -35,10 +35,7 @@ class Configuration: @classmethod def from_design_ref(cls, design_ref): - response = requests.get(design_ref) - response.raise_for_status() - - documents = list(yaml.safe_load_all(response.text)) + documents = get_documents(design_ref) validation.check_schemas(documents) return cls(documents=documents) diff --git a/promenade/design_ref.py b/promenade/design_ref.py new file mode 100644 index 00000000..cbee1f68 --- /dev/null +++ b/promenade/design_ref.py @@ -0,0 +1,42 @@ +from . import logging +from oslo_config import cfg +import keystoneauth1.identity.v3 +import keystoneauth1.session +import requests +import yaml + +LOG = logging.getLogger(__name__) + +__all__ = ['get_documents'] + +_DECKHAND_PREFIX = 'deckhand+' + +DH_TIMEOUT = 10 * 60 # 10 Minute timeout for fetching from Deckhand. + + +def get_documents(design_ref): + LOG.debug('Fetching design_ref="%s"', design_ref) + if design_ref.startswith(_DECKHAND_PREFIX): + response = _get_from_deckhand(design_ref) + else: + response = _get_from_basic_web(design_ref) + LOG.debug('Got response for design_ref="%s"', design_ref) + + response.raise_for_status() + + return list(yaml.safe_load_all(response.text)) + + +def _get_from_basic_web(design_ref): + return requests.get(design_ref) + + +def _get_from_deckhand(design_ref): + keystone_args = {} + for attr in ('auth_url', 'password', 'project_domain_name', 'project_name', + 'username', 'user_domain_name'): + keystone_args[attr] = cfg.CONF.get('keystone_authtoken', {}).get(attr) + auth = keystoneauth1.identity.v3.Password(**keystone_args) + session = keystoneauth1.session.Session(auth=auth) + + return session.get(design_ref[len(_DECKHAND_PREFIX):], timeout=DH_TIMEOUT) diff --git a/promenade/exceptions.py b/promenade/exceptions.py index be2d40e2..5a2cd1e3 100644 --- a/promenade/exceptions.py +++ b/promenade/exceptions.py @@ -215,7 +215,7 @@ class PromenadeException(Exception): if self.trace or debug: LOG.exception(self.description) else: - LOG.error(self.description) + LOG.error(self.title + (self.description or '')) class ApiError(PromenadeException): @@ -238,6 +238,7 @@ class InvalidFormatError(PromenadeException): class ValidationException(PromenadeException): title = 'Validation Error' + status = falcon.HTTP_400 def massage_error_list(error_list, placeholder_description): diff --git a/promenade/options.py b/promenade/options.py index 4678eed7..9d528423 100644 --- a/promenade/options.py +++ b/promenade/options.py @@ -1,5 +1,33 @@ from oslo_config import cfg +import keystoneauth1.loading OPTIONS = [] -cfg.CONF.register_opts(OPTIONS) + +def setup(disable=None): + if disable is None: + disable = [] + else: + disable = disable.split() + + for name, func in GROUPS.items(): + if name not in disable: + func() + + cfg.CONF([], project='promenade') + + +def register_application(): + cfg.CONF.register_opts(OPTIONS) + + +def register_keystone_auth(): + cfg.CONF.register_opts( + keystoneauth1.loading.get_auth_plugin_conf_options('password'), + group='keystone_authtoken') + + +GROUPS = { + 'promenade': register_application, + 'keystone': register_keystone_auth, +} diff --git a/promenade/pki.py b/promenade/pki.py index e05e9e45..63866ebc 100644 --- a/promenade/pki.py +++ b/promenade/pki.py @@ -154,10 +154,11 @@ class PKI: 'metadata': { 'schema': 'metadata/Document/v1', 'name': name, - 'layerinDefinition': { + 'layeringDefinition': { 'abstract': False, 'layer': 'site', }, + 'storagePolicy': 'cleartext', }, 'data': block_literal(data), } diff --git a/promenade/promenade.py b/promenade/promenade.py index 558d8271..1fbebd17 100644 --- a/promenade/promenade.py +++ b/promenade/promenade.py @@ -11,15 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from oslo_config import cfg from promenade.control import api -from promenade import options # noqa +from promenade import options from promenade import logging from promenade import policy -def start_promenade(): - cfg.CONF() +def start_promenade(disable=""): + options.setup(disable=disable) + # Setup root logger logging.setup(verbose=True) @@ -33,7 +33,4 @@ def start_promenade(): # Initialization compatible with PasteDeploy def paste_start_promenade(global_conf, **kwargs): - return promenade - - -promenade = start_promenade() + return start_promenade(**kwargs) diff --git a/requirements-direct.txt b/requirements-direct.txt index 4d90217a..2611b77d 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -3,6 +3,7 @@ falcon==1.2.0 jinja2==2.9.6 jsonpath-ng==1.4.3 jsonschema==2.6.0 +keystoneauth1==3.2.0 keystonemiddleware==4.17.0 kubernetes==3.0.0 oslo.context==2.19.2 diff --git a/tests/unit/api/test_health_api.py b/tests/unit/api/test_health_api.py index 32b2bf1f..51f9be25 100644 --- a/tests/unit/api/test_health_api.py +++ b/tests/unit/api/test_health_api.py @@ -17,12 +17,12 @@ from falcon import testing import pytest from promenade.control import health_api -from promenade.promenade import promenade +from promenade import promenade @pytest.fixture() def client(): - return testing.TestClient(promenade) + return testing.TestClient(promenade.start_promenade(disable='keystone')) def test_get_health(client): diff --git a/tests/unit/api/test_versions.py b/tests/unit/api/test_versions.py index 00d0c071..c1977058 100644 --- a/tests/unit/api/test_versions.py +++ b/tests/unit/api/test_versions.py @@ -17,12 +17,12 @@ from falcon import testing import pytest from promenade.control.api import VersionsResource -from promenade.promenade import promenade +from promenade import promenade @pytest.fixture() def client(): - return testing.TestClient(promenade) + return testing.TestClient(promenade.start_promenade(disable='keystone')) def test_get_versions(client): diff --git a/tools/g2/lib/openstack.sh b/tools/g2/lib/openstack.sh index ba478e5d..d9dd70d5 100644 --- a/tools/g2/lib/openstack.sh +++ b/tools/g2/lib/openstack.sh @@ -26,6 +26,10 @@ EOBODY rsync_cmd "${TEMP_DIR}/${REQUEST_BODY_PATH}" "${VIA}:/root/${REQUEST_BODY_PATH}" ssh_cmd "${VIA}" curl -isS \ + --fail \ + --max-time 60 \ + --retry 10 \ + --retry-delay 15 \ -H 'Content-Type: application/json' \ -d "@/root/${REQUEST_BODY_PATH}" \ "${KEYSTONE_URL}/v3/auth/tokens" | grep 'X-Subject-Token' | awk '{print $2}' | sed "s;';;g" | sed "s;\r;;g" diff --git a/tools/g2/lib/registry.sh b/tools/g2/lib/registry.sh index 9aa51f44..e41b9a79 100644 --- a/tools/g2/lib/registry.sh +++ b/tools/g2/lib/registry.sh @@ -7,7 +7,7 @@ registry_down() { } registry_list_images() { - FILES=($(find "$(config_configuration)" -type f -name '*.yaml')) + FILES=($(echo "$(config_configuration)" | xargs -n 1 -I DIRNAME find DIRNAME -type f -name '*.yaml')) HOSTNAME_REGEX='[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}' DOMAIN_NAME_REGEX="${HOSTNAME_REGEX}(\.${HOSTNAME_REGEX})*" diff --git a/tools/g2/manifests/conformance.json b/tools/g2/manifests/conformance.json index 5cca5d3a..02309f2a 100644 --- a/tools/g2/manifests/conformance.json +++ b/tools/g2/manifests/conformance.json @@ -1,6 +1,7 @@ { "configuration": [ - "examples/basic" + "examples/basic", + "promenade/schemas" ], "stages": [ { diff --git a/tools/g2/manifests/genesis.json b/tools/g2/manifests/genesis.json index 3dc48668..572f6506 100644 --- a/tools/g2/manifests/genesis.json +++ b/tools/g2/manifests/genesis.json @@ -1,6 +1,7 @@ { "configuration": [ - "examples/complete" + "examples/complete", + "promenade/schemas" ], "stages": [ { diff --git a/tools/g2/manifests/integration.json b/tools/g2/manifests/integration.json index 1adfb1b5..9b1139fa 100644 --- a/tools/g2/manifests/integration.json +++ b/tools/g2/manifests/integration.json @@ -1,6 +1,7 @@ { "configuration": [ - "examples/complete" + "examples/complete", + "promenade/schemas" ], "stages": [ { @@ -27,12 +28,20 @@ "name": "Genesis", "script": "genesis.sh" }, + { + "name": "Load Site Configuration", + "script": "load-site-config.sh", + "arguments": [ + "-v", "n0" + ] + }, { "name": "Join Master", "script": "join-nodes.sh", "arguments": [ "-v", "n0", "-t", + "-d", "1", "-n", "n1", "-l", "calico-etcd=enabled", "-l", "kubernetes-apiserver=enabled", diff --git a/tools/g2/manifests/resiliency.json b/tools/g2/manifests/resiliency.json index d8a0af85..fb108d30 100644 --- a/tools/g2/manifests/resiliency.json +++ b/tools/g2/manifests/resiliency.json @@ -1,6 +1,7 @@ { "configuration": [ - "examples/basic" + "examples/basic", + "promenade/schemas" ], "stages": [ { diff --git a/tools/g2/manifests/smoke.json b/tools/g2/manifests/smoke.json index db5c4b8c..1a9bb9a4 100644 --- a/tools/g2/manifests/smoke.json +++ b/tools/g2/manifests/smoke.json @@ -1,6 +1,7 @@ { "configuration": [ - "examples/complete" + "examples/basic", + "promenade/schemas" ], "stages": [ { diff --git a/tools/g2/stages/build-scripts.sh b/tools/g2/stages/build-scripts.sh index 8d3b7f49..97ca064c 100755 --- a/tools/g2/stages/build-scripts.sh +++ b/tools/g2/stages/build-scripts.sh @@ -20,4 +20,5 @@ docker run --rm -t \ -o scripts \ config/*.yaml +mkdir -p "${TEMP_DIR}/nginx/" cat "${TEMP_DIR}"/config/*.yaml > "${TEMP_DIR}/nginx/promenade.yaml" diff --git a/tools/g2/stages/generate-certificates.sh b/tools/g2/stages/generate-certificates.sh index 1a8b2d80..3ca95a25 100755 --- a/tools/g2/stages/generate-certificates.sh +++ b/tools/g2/stages/generate-certificates.sh @@ -7,10 +7,11 @@ source "${GATE_UTILS}" OUTPUT_DIR="${TEMP_DIR}/config" mkdir -p "${OUTPUT_DIR}" chmod 777 "${OUTPUT_DIR}" +OUTPUT_FILE="${OUTPUT_DIR}/combined.yaml" for source_dir in $(config_configuration); do log Copying configuration from "${source_dir}" - cp "${WORKSPACE}/${source_dir}"/*.yaml "${OUTPUT_DIR}" + cat "${WORKSPACE}/${source_dir}"/*.yaml >> "${OUTPUT_FILE}" done registry_replace_references "${OUTPUT_DIR}"/*.yaml diff --git a/tools/g2/stages/join-nodes.sh b/tools/g2/stages/join-nodes.sh index 1eae7d00..551257e9 100755 --- a/tools/g2/stages/join-nodes.sh +++ b/tools/g2/stages/join-nodes.sh @@ -9,12 +9,17 @@ declare -a LABELS declare -a NODES GET_KEYSTONE_TOKEN=0 +USE_DECKHAND=0 -while getopts "e:l:n:tv:" opt; do +while getopts "d:e:l:n:tv:" opt; do case "${opt}" in e) ETCD_CLUSTERS+=("${OPTARG}") ;; + d) + USE_DECKHAND=1 + DECKHAND_REVISION=${OPTARG} + ;; l) LABELS+=("${OPTARG}") ;; @@ -58,7 +63,11 @@ render_curl_url() { done BASE_URL="${BASE_PROM_URL}/api/v1.0/join-scripts" - DESIGN_REF="design_ref=http://192.168.77.1:7777/promenade.yaml" + if [[ ${USE_DECKHAND} == 1 ]]; then + DESIGN_REF="design_ref=deckhand%2Bhttp://deckhand-int.ucp.svc.cluster.local:9000/api/v1.0/revisions/${DECKHAND_REVISION}/rendered-documents" + else + DESIGN_REF="design_ref=http://192.168.77.1:7777/promenade.yaml" + fi HOST_PARAMS="hostname=${NAME}&ip=$(config_vm_ip "${NAME}")" echo "${BASE_URL}?${DESIGN_REF}&${HOST_PARAMS}${LABEL_PARAMS}" diff --git a/tools/g2/stages/load-site-config.sh b/tools/g2/stages/load-site-config.sh new file mode 100755 index 00000000..f3ab79e6 --- /dev/null +++ b/tools/g2/stages/load-site-config.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -eu + +source "${GATE_UTILS}" + +while getopts "v:" opt; do + case "${opt}" in + v) + VIA=${OPTARG} + ;; + *) + echo "Unknown option" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ $# -gt 0 ]; then + echo "Unknown arguments specified: ${*}" + exit 1 +fi + +TOKEN=$(os_ks_get_token "${VIA}") + +DECKHAND_URL=http://deckhand-int.ucp.svc.cluster.local:9000/api/v1.0/buckets/prom/documents + +rsync_cmd "${TEMP_DIR}/nginx/promenade.yaml" "${VIA}:/root/promenade/promenade.yaml" +ssh_cmd "${VIA}" curl -v \ + --fail \ + --max-time 300 \ + --retry 10 \ + --retry-delay 15 \ + -H "X-Auth-Token: ${TOKEN}" \ + -H "Content-Type: application/x-yaml" \ + -T "/root/promenade/promenade.yaml" \ + "${DECKHAND_URL}" diff --git a/tools/gate/config-templates/LayeringPolicy.yaml b/tools/gate/config-templates/LayeringPolicy.yaml new file mode 100644 index 00000000..46ae0c58 --- /dev/null +++ b/tools/gate/config-templates/LayeringPolicy.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/LayeringPolicy/v1 +metadata: + schema: metadata/Control/v1 + name: layering-policy +data: + layerOrder: + - global + - type + - site +... diff --git a/tox.ini b/tox.ini index 03ecf7e7..d04f616c 100644 --- a/tox.ini +++ b/tox.ini @@ -29,6 +29,7 @@ commands = yapf -ir {toxinidir}/promenade {toxinidir}/tests [testenv:freeze] +basepython=python3.5 deps = -r{toxinidir}/requirements-direct.txt recreate = True whitelist_externals = sh