From 7917237ae0e5e4f8ea2bad5ef26db93e22542f98 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Fri, 12 Jan 2018 16:43:20 -0600 Subject: [PATCH] Migrate to DH-managed config files Use the Deckhand engine module directly to manage local configuration files during CLI usage. Note: not doing document validation as DH currently requires schemas to be sourced from the database. Simple schema validation in place. - Layering/substitution - Schema validation based on DataSchema documents in payload - Add deckhand to requirements A few tooling updates - concatenate test & schema yaml files into a single file to avoid name conflicts - make nginx directory in build-scripts stage Change-Id: I2d56244f01c58052f14331bc09fd5843d4c95292 --- examples/basic/Docker.yaml | 1 + examples/basic/Kubelet.yaml | 1 + examples/basic/KubernetesNetwork.yaml | 1 + examples/basic/armada-resources.yaml | 252 +++++++++--------- examples/complete/Docker.yaml | 1 - examples/complete/Kubelet.yaml | 1 - examples/complete/KubernetesNetwork.yaml | 1 - examples/complete/armada-resources.yaml | 252 +++++++++--------- promenade/builder.py | 2 + promenade/cli.py | 5 +- promenade/config.py | 41 ++- promenade/logging.py | 5 + promenade/validation.py | 35 ++- requirements-direct.txt | 1 + requirements-frozen.txt | 57 +++- tools/g2/manifests/smoke.json | 2 +- .../bootstrap-armada-config.yaml | 19 ++ .../gate/config-templates/genesis-config.yaml | 1 + .../config-templates/joining-host-config.yaml | 4 + tools/gate/config-templates/site-config.yaml | 4 + 20 files changed, 406 insertions(+), 280 deletions(-) diff --git a/examples/basic/Docker.yaml b/examples/basic/Docker.yaml index 9886aeeb..9463e9f9 100644 --- a/examples/basic/Docker.yaml +++ b/examples/basic/Docker.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: config: insecure-registries: diff --git a/examples/basic/Kubelet.yaml b/examples/basic/Kubelet.yaml index def6c18b..41b84ce4 100644 --- a/examples/basic/Kubelet.yaml +++ b/examples/basic/Kubelet.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: arguments: - --cni-bin-dir=/opt/cni/bin diff --git a/examples/basic/KubernetesNetwork.yaml b/examples/basic/KubernetesNetwork.yaml index b5755010..9c3d0373 100644 --- a/examples/basic/KubernetesNetwork.yaml +++ b/examples/basic/KubernetesNetwork.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: dns: cluster_domain: cluster.local diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index d644a3be..f2c2e1fd 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -124,23 +124,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: '$.values.secrets.tls.ca' + path: '.values.secrets.tls.ca' - src: schema: deckhand/Certificate/v1 name: proxy - path: $ + path: . dest: - path: '$.values.secrets.tls.cert' + path: '.values.secrets.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: proxy - path: $ + path: . dest: - path: '$.values.secrets.tls.key' + path: '.values.secrets.tls.key' data: chart_name: proxy release: kubernetes-proxy @@ -180,147 +180,147 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: calico-etcd - path: $ + path: . dest: - path: '$.values.secrets.tls.client.ca' + path: '.values.secrets.tls.client.ca' - src: schema: deckhand/CertificateAuthority/v1 name: calico-etcd-peer - path: $ + path: . dest: - path: '$.values.secrets.tls.peer.ca' + path: '.values.secrets.tls.peer.ca' - src: schema: deckhand/Certificate/v1 name: calico-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.cert' + path: '.values.secrets.anchor.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.key' + path: '.values.secrets.anchor.tls.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n0 - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.cert' + path: '.values.nodes[0].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n0 - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.key' + path: '.values.nodes[0].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n0-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.cert' + path: '.values.nodes[0].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n0-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.key' + path: '.values.nodes[0].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n1 - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.cert' + path: '.values.nodes[1].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n1 - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.key' + path: '.values.nodes[1].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n1-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.cert' + path: '.values.nodes[1].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n1-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.key' + path: '.values.nodes[1].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n2 - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.cert' + path: '.values.nodes[2].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n2 - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.key' + path: '.values.nodes[2].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n2-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.cert' + path: '.values.nodes[2].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n2-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.key' + path: '.values.nodes[2].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n3 - path: $ + path: . dest: - path: '$.values.nodes[3].tls.client.cert' + path: '.values.nodes[3].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n3 - path: $ + path: . dest: - path: '$.values.nodes[3].tls.client.key' + path: '.values.nodes[3].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n3-peer - path: $ + path: . dest: - path: '$.values.nodes[3].tls.peer.cert' + path: '.values.nodes[3].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n3-peer - path: $ + path: . dest: - path: '$.values.nodes[3].tls.peer.key' + path: '.values.nodes[3].tls.peer.key' data: chart_name: etcd @@ -424,23 +424,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: calico-etcd - path: $ + path: . dest: - path: '$.values.etcd.tls.ca' + path: '.values.etcd.tls.ca' - src: schema: deckhand/Certificate/v1 name: calico-node - path: $ + path: . dest: - path: '$.values.etcd.tls.cert' + path: '.values.etcd.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-node - path: $ + path: . dest: - path: '$.values.etcd.tls.key' + path: '.values.etcd.tls.key' data: chart_name: calico release: calico @@ -487,23 +487,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: '$.values.tls.ca' + path: '.values.tls.ca' - src: schema: deckhand/Certificate/v1 name: coredns - path: $ + path: . dest: - path: '$.values.tls.cert' + path: '.values.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: coredns - path: $ + path: . dest: - path: '$.values.tls.key' + path: '.values.tls.key' data: chart_name: coredns release: coredns @@ -567,52 +567,52 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: $.values.secrets.tls.ca + path: .values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: apiserver - path: $ + path: . dest: - path: $.values.secrets.tls.cert + path: .values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: apiserver - path: $ + path: . dest: - path: $.values.secrets.tls.key + path: .values.secrets.tls.key - src: schema: deckhand/CertificateAuthority/v1 name: kubernetes-etcd - path: $ + path: . dest: - path: $.values.secrets.etcd.tls.ca + path: .values.secrets.etcd.tls.ca - src: schema: deckhand/Certificate/v1 name: apiserver-etcd - path: $ + path: . dest: - path: $.values.secrets.etcd.tls.cert + path: .values.secrets.etcd.tls.cert - src: schema: deckhand/CertificateKey/v1 name: apiserver-etcd - path: $ + path: . dest: - path: $.values.secrets.etcd.tls.key + path: .values.secrets.etcd.tls.key - src: schema: deckhand/PublicKey/v1 name: service-account - path: $ + path: . dest: - path: $.values.secrets.service_account.public_key + path: .values.secrets.service_account.public_key data: chart_name: apiserver @@ -668,31 +668,31 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: $.values.secrets.tls.ca + path: .values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: controller-manager - path: $ + path: . dest: - path: $.values.secrets.tls.cert + path: .values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: controller-manager - path: $ + path: . dest: - path: $.values.secrets.tls.key + path: .values.secrets.tls.key - src: schema: deckhand/PrivateKey/v1 name: service-account - path: $ + path: . dest: - path: $.values.secrets.service_account.private_key + path: .values.secrets.service_account.private_key data: chart_name: controller_manager @@ -740,23 +740,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: $.values.secrets.tls.ca + path: .values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: scheduler - path: $ + path: . dest: - path: $.values.secrets.tls.cert + path: .values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: scheduler - path: $ + path: . dest: - path: $.values.secrets.tls.key + path: .values.secrets.tls.key data: chart_name: scheduler @@ -802,147 +802,147 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes-etcd - path: $ + path: . dest: - path: '$.values.secrets.tls.client.ca' + path: '.values.secrets.tls.client.ca' - src: schema: deckhand/CertificateAuthority/v1 name: kubernetes-etcd-peer - path: $ + path: . dest: - path: '$.values.secrets.tls.peer.ca' + path: '.values.secrets.tls.peer.ca' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.cert' + path: '.values.secrets.anchor.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.key' + path: '.values.secrets.anchor.tls.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n0 - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.cert' + path: '.values.nodes[0].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n0 - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.key' + path: '.values.nodes[0].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n0-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.cert' + path: '.values.nodes[0].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n0-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.key' + path: '.values.nodes[0].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n1 - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.cert' + path: '.values.nodes[1].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n1 - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.key' + path: '.values.nodes[1].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n1-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.cert' + path: '.values.nodes[1].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n1-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.key' + path: '.values.nodes[1].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n2 - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.cert' + path: '.values.nodes[2].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n2 - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.key' + path: '.values.nodes[2].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n2-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.cert' + path: '.values.nodes[2].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n2-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.key' + path: '.values.nodes[2].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n3 - path: $ + path: . dest: - path: '$.values.nodes[3].tls.client.cert' + path: '.values.nodes[3].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n3 - path: $ + path: . dest: - path: '$.values.nodes[3].tls.client.key' + path: '.values.nodes[3].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n3-peer - path: $ + path: . dest: - path: '$.values.nodes[3].tls.peer.cert' + path: '.values.nodes[3].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n3-peer - path: $ + path: . dest: - path: '$.values.nodes[3].tls.peer.key' + path: '.values.nodes[3].tls.peer.key' data: chart_name: etcd diff --git a/examples/complete/Docker.yaml b/examples/complete/Docker.yaml index 9463e9f9..9886aeeb 100644 --- a/examples/complete/Docker.yaml +++ b/examples/complete/Docker.yaml @@ -6,7 +6,6 @@ metadata: layeringDefinition: abstract: false layer: site - storagePolicy: cleartext data: config: insecure-registries: diff --git a/examples/complete/Kubelet.yaml b/examples/complete/Kubelet.yaml index 41b84ce4..def6c18b 100644 --- a/examples/complete/Kubelet.yaml +++ b/examples/complete/Kubelet.yaml @@ -6,7 +6,6 @@ metadata: layeringDefinition: abstract: false layer: site - storagePolicy: cleartext data: arguments: - --cni-bin-dir=/opt/cni/bin diff --git a/examples/complete/KubernetesNetwork.yaml b/examples/complete/KubernetesNetwork.yaml index 9c3d0373..b5755010 100644 --- a/examples/complete/KubernetesNetwork.yaml +++ b/examples/complete/KubernetesNetwork.yaml @@ -6,7 +6,6 @@ metadata: layeringDefinition: abstract: false layer: site - storagePolicy: cleartext data: dns: cluster_domain: cluster.local diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml index 7b8127f9..a9c166d8 100644 --- a/examples/complete/armada-resources.yaml +++ b/examples/complete/armada-resources.yaml @@ -168,23 +168,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: '$.values.secrets.tls.ca' + path: '.values.secrets.tls.ca' - src: schema: deckhand/Certificate/v1 name: proxy - path: $ + path: . dest: - path: '$.values.secrets.tls.cert' + path: '.values.secrets.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: proxy - path: $ + path: . dest: - path: '$.values.secrets.tls.key' + path: '.values.secrets.tls.key' data: chart_name: proxy release: kubernetes-proxy @@ -225,147 +225,147 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: calico-etcd - path: $ + path: . dest: - path: '$.values.secrets.tls.client.ca' + path: '.values.secrets.tls.client.ca' - src: schema: deckhand/CertificateAuthority/v1 name: calico-etcd-peer - path: $ + path: . dest: - path: '$.values.secrets.tls.peer.ca' + path: '.values.secrets.tls.peer.ca' - src: schema: deckhand/Certificate/v1 name: calico-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.cert' + path: '.values.secrets.anchor.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.key' + path: '.values.secrets.anchor.tls.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n0 - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.cert' + path: '.values.nodes[0].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n0 - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.key' + path: '.values.nodes[0].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n0-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.cert' + path: '.values.nodes[0].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n0-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.key' + path: '.values.nodes[0].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n1 - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.cert' + path: '.values.nodes[1].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n1 - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.key' + path: '.values.nodes[1].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n1-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.cert' + path: '.values.nodes[1].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n1-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.key' + path: '.values.nodes[1].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n2 - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.cert' + path: '.values.nodes[2].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n2 - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.key' + path: '.values.nodes[2].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n2-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.cert' + path: '.values.nodes[2].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n2-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.key' + path: '.values.nodes[2].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n3 - path: $ + path: . dest: - path: '$.values.nodes[3].tls.client.cert' + path: '.values.nodes[3].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n3 - path: $ + path: . dest: - path: '$.values.nodes[3].tls.client.key' + path: '.values.nodes[3].tls.client.key' - src: schema: deckhand/Certificate/v1 name: calico-etcd-n3-peer - path: $ + path: . dest: - path: '$.values.nodes[3].tls.peer.cert' + path: '.values.nodes[3].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-etcd-n3-peer - path: $ + path: . dest: - path: '$.values.nodes[3].tls.peer.key' + path: '.values.nodes[3].tls.peer.key' data: chart_name: etcd @@ -470,23 +470,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: calico-etcd - path: $ + path: . dest: - path: '$.values.etcd.tls.ca' + path: '.values.etcd.tls.ca' - src: schema: deckhand/Certificate/v1 name: calico-node - path: $ + path: . dest: - path: '$.values.etcd.tls.cert' + path: '.values.etcd.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: calico-node - path: $ + path: . dest: - path: '$.values.etcd.tls.key' + path: '.values.etcd.tls.key' data: chart_name: calico release: calico @@ -534,23 +534,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: '$.values.tls.ca' + path: '.values.tls.ca' - src: schema: deckhand/Certificate/v1 name: coredns - path: $ + path: . dest: - path: '$.values.tls.cert' + path: '.values.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: coredns - path: $ + path: . dest: - path: '$.values.tls.key' + path: '.values.tls.key' data: chart_name: coredns release: coredns @@ -614,52 +614,52 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: $.values.secrets.tls.ca + path: .values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: apiserver - path: $ + path: . dest: - path: $.values.secrets.tls.cert + path: .values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: apiserver - path: $ + path: . dest: - path: $.values.secrets.tls.key + path: .values.secrets.tls.key - src: schema: deckhand/CertificateAuthority/v1 name: kubernetes-etcd - path: $ + path: . dest: - path: $.values.secrets.etcd.tls.ca + path: .values.secrets.etcd.tls.ca - src: schema: deckhand/Certificate/v1 name: apiserver-etcd - path: $ + path: . dest: - path: $.values.secrets.etcd.tls.cert + path: .values.secrets.etcd.tls.cert - src: schema: deckhand/CertificateKey/v1 name: apiserver-etcd - path: $ + path: . dest: - path: $.values.secrets.etcd.tls.key + path: .values.secrets.etcd.tls.key - src: schema: deckhand/PublicKey/v1 name: service-account - path: $ + path: . dest: - path: $.values.secrets.service_account.public_key + path: .values.secrets.service_account.public_key data: chart_name: apiserver @@ -715,31 +715,31 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: $.values.secrets.tls.ca + path: .values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: controller-manager - path: $ + path: . dest: - path: $.values.secrets.tls.cert + path: .values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: controller-manager - path: $ + path: . dest: - path: $.values.secrets.tls.key + path: .values.secrets.tls.key - src: schema: deckhand/PrivateKey/v1 name: service-account - path: $ + path: . dest: - path: $.values.secrets.service_account.private_key + path: .values.secrets.service_account.private_key data: chart_name: controller_manager @@ -787,23 +787,23 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes - path: $ + path: . dest: - path: $.values.secrets.tls.ca + path: .values.secrets.tls.ca - src: schema: deckhand/Certificate/v1 name: scheduler - path: $ + path: . dest: - path: $.values.secrets.tls.cert + path: .values.secrets.tls.cert - src: schema: deckhand/CertificateKey/v1 name: scheduler - path: $ + path: . dest: - path: $.values.secrets.tls.key + path: .values.secrets.tls.key data: chart_name: scheduler @@ -849,147 +849,147 @@ metadata: src: schema: deckhand/CertificateAuthority/v1 name: kubernetes-etcd - path: $ + path: . dest: - path: '$.values.secrets.tls.client.ca' + path: '.values.secrets.tls.client.ca' - src: schema: deckhand/CertificateAuthority/v1 name: kubernetes-etcd-peer - path: $ + path: . dest: - path: '$.values.secrets.tls.peer.ca' + path: '.values.secrets.tls.peer.ca' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.cert' + path: '.values.secrets.anchor.tls.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-anchor - path: $ + path: . dest: - path: '$.values.secrets.anchor.tls.key' + path: '.values.secrets.anchor.tls.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n0 - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.cert' + path: '.values.nodes[0].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n0 - path: $ + path: . dest: - path: '$.values.nodes[0].tls.client.key' + path: '.values.nodes[0].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n0-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.cert' + path: '.values.nodes[0].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n0-peer - path: $ + path: . dest: - path: '$.values.nodes[0].tls.peer.key' + path: '.values.nodes[0].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n1 - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.cert' + path: '.values.nodes[1].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n1 - path: $ + path: . dest: - path: '$.values.nodes[1].tls.client.key' + path: '.values.nodes[1].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n1-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.cert' + path: '.values.nodes[1].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n1-peer - path: $ + path: . dest: - path: '$.values.nodes[1].tls.peer.key' + path: '.values.nodes[1].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n2 - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.cert' + path: '.values.nodes[2].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n2 - path: $ + path: . dest: - path: '$.values.nodes[2].tls.client.key' + path: '.values.nodes[2].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n2-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.cert' + path: '.values.nodes[2].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n2-peer - path: $ + path: . dest: - path: '$.values.nodes[2].tls.peer.key' + path: '.values.nodes[2].tls.peer.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n3 - path: $ + path: . dest: - path: '$.values.nodes[3].tls.client.cert' + path: '.values.nodes[3].tls.client.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n3 - path: $ + path: . dest: - path: '$.values.nodes[3].tls.client.key' + path: '.values.nodes[3].tls.client.key' - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-n3-peer - path: $ + path: . dest: - path: '$.values.nodes[3].tls.peer.cert' + path: '.values.nodes[3].tls.peer.cert' - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-n3-peer - path: $ + path: . dest: - path: '$.values.nodes[3].tls.peer.key' + path: '.values.nodes[3].tls.peer.key' data: chart_name: etcd diff --git a/promenade/builder.py b/promenade/builder.py index 47e686c2..394e14f7 100644 --- a/promenade/builder.py +++ b/promenade/builder.py @@ -112,8 +112,10 @@ class Builder: def _fetch_tar_content(*, url, path): + LOG.debug('Fetching url=%s (tar path=%s)', url, path) response = requests.get(url) response.raise_for_status() + LOG.debug('Finished downloading url=%s (tar path=%s)', url, path) f = io.BytesIO(response.content) tf = tarfile.open(fileobj=f, mode='r') buf_reader = tf.extractfile(path) diff --git a/promenade/cli.py b/promenade/cli.py index 8dea4bcf..5c6289c4 100644 --- a/promenade/cli.py +++ b/promenade/cli.py @@ -56,7 +56,10 @@ def genereate_certs(*, calico_etcd_service_ip, config_files, output_dir): debug = _debug() try: c = config.Configuration.from_streams( - debug=debug, streams=config_files, substitute=False) + debug=debug, + streams=config_files, + substitute=False, + validate=False) g = generator.Generator( c, calico_etcd_service_ip=calico_etcd_service_ip) g.generate(output_dir) diff --git a/promenade/config.py b/promenade/config.py index 25aee841..8f4b6493 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -5,15 +5,32 @@ import jinja2 import jsonpath_ng import yaml +from deckhand.engine import layering + __all__ = ['Configuration'] LOG = logging.getLogger(__name__) class Configuration: - def __init__(self, *, documents, debug=False, substitute=True): + def __init__(self, + *, + documents, + debug=False, + substitute=True, + validate=True): + LOG.info("Parsing document schemas.") + schema_set = validation.load_schemas_from_docs(documents) + LOG.info("Parsed %d document schemas." % len(schema_set)) + LOG.info("Building config from %d documents." % len(documents)) if substitute: - documents = _substitute(documents) + LOG.info("Rendering documents via Deckhand engine.") + deckhand_eng = layering.DocumentLayering( + documents, substitution_sources=documents) + documents = [dict(d) for d in deckhand_eng.render()] + LOG.info("Deckhand engine returned %d documents." % len(documents)) + if validate: + validation.check_schemas(documents, schemas=schema_set) self.debug = debug self.documents = documents @@ -25,20 +42,18 @@ class Configuration: if stream_name is not None: LOG.info('Loading documents from %s', stream_name) stream_documents = list(yaml.safe_load_all(stream)) - validation.check_schemas(stream_documents) if stream_name is not None: - LOG.info('Successfully validated documents from %s', - stream_name) + LOG.info('Successfully loaded %d documents from %s', + len(stream_documents), stream_name) documents.extend(stream_documents) return cls(documents=documents, **kwargs) @classmethod - def from_design_ref(cls, design_ref): + def from_design_ref(cls, design_ref, **kwargs): documents = get_documents(design_ref) - validation.check_schemas(documents) - return cls(documents=documents) + return cls(documents=documents, **kwargs) def __getitem__(self, path): value = self.get_path(path) @@ -86,7 +101,10 @@ class Configuration: LOG.debug('Excluding schema=%s metadata.name=%s', document['schema'], _mg(document, 'name')) return Configuration( - debug=self.debug, documents=documents, substitute=False) + debug=self.debug, + documents=documents, + substitute=False, + validate=False) def extract_node_config(self, name): LOG.debug('Extracting node config for %s.', name) @@ -105,7 +123,10 @@ class Configuration: else: documents.append(document) return Configuration( - debug=self.debug, documents=documents, substitute=False) + debug=self.debug, + documents=documents, + substitute=False, + validate=False) @property def kubelet_name(self): diff --git a/promenade/logging.py b/promenade/logging.py index d8f02ae2..8342eed2 100644 --- a/promenade/logging.py +++ b/promenade/logging.py @@ -34,6 +34,11 @@ DEFAULT_CONFIG = { }, }, 'loggers': { + 'deckhand': { + 'handlers': ['default'], + 'level': 'INFO', + 'propagate': False, + }, 'promenade': { 'handlers': ['default'], 'level': 'INFO', diff --git a/promenade/validation.py b/promenade/validation.py index ed73ef43..b8778a84 100644 --- a/promenade/validation.py +++ b/promenade/validation.py @@ -40,13 +40,15 @@ def check_design(config): raise exceptions.ValidationException() -def check_schemas(documents): +def check_schemas(documents, schemas=None): + if not schemas: + schemas = load_schemas_from_docs(documents) for document in documents: - check_schema(document) + check_schema(document, schemas=schemas) -def check_schema(document): - if type(document) != dict: +def check_schema(document, schemas=None): + if not isinstance(document, dict): LOG.error('Non-dictionary document passed to schema validation.') return @@ -55,9 +57,11 @@ def check_schema(document): LOG.debug('Validating schema for schema=%s metadata.name=%s', schema_name, document.get('metadata', {}).get('name', '')) - if schema_name in SCHEMAS: + schema_set = SCHEMAS if schemas is None else schemas + + if schema_name in schema_set: try: - jsonschema.validate(document.get('data'), SCHEMAS[schema_name]) + jsonschema.validate(document.get('data'), schema_set[schema_name]) except jsonschema.ValidationError as e: raise exceptions.ValidationException(str(e)) else: @@ -67,6 +71,25 @@ def check_schema(document): SCHEMAS = {} +def load_schemas_from_docs(doc_set): + ''' + Fills the cache of known schemas from the document set + ''' + SCHEMA_SCHEMA = "deckhand/DataSchema/v1" + + schema_set = dict() + for document in doc_set: + if document.get('schema', '') == SCHEMA_SCHEMA: + name = document['metadata']['name'] + LOG.debug("Found schema for %s." % name) + if name in schema_set: + raise RuntimeError('Duplicate schema specified for: %s' % name) + + schema_set[name] = document['data'] + + return schema_set + + def _load_schemas(): ''' Fills the cache of known schemas diff --git a/requirements-direct.txt b/requirements-direct.txt index 2611b77d..19734a87 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -13,3 +13,4 @@ pbr==3.0.1 pyyaml==3.12 requests==2.18.4 uwsgi==2.0.15 +git+https://github.com/att-comdev/deckhand.git@master#egg=deckhand diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 1b0c6436..f8ac0394 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,54 +1,97 @@ +alembic==0.9.6 +amqp==2.2.2 Babel==2.5.1 cachetools==2.0.1 certifi==2017.11.5 chardet==3.0.4 click==6.7 +cliff==2.10.0 +cmd2==0.7.9 +contextlib2==0.5.5 debtcollector==1.19.0 +git+https://github.com/att-comdev/deckhand.git@master#egg=deckhand decorator==4.1.2 +dogpile.cache==0.6.4 +enum-compat==0.0.2 +eventlet==0.20.0 falcon==1.2.0 -google-auth==1.2.1 +fasteners==0.14.1 +flake8==2.5.5 +futurist==1.6.0 +google-auth==1.3.0 +greenlet==0.4.12 +hacking==1.0.0 idna==2.6 -ipaddress==1.0.18 +ipaddress==1.0.19 iso8601==0.1.12 Jinja2==2.9.6 jsonpath-ng==1.4.3 jsonschema==2.6.0 keystoneauth1==3.3.0 keystonemiddleware==4.17.0 +kombu==4.1.0 kubernetes==3.0.0 +Mako==1.0.7 MarkupSafe==1.0 +mccabe==0.2.1 monotonic==1.4 -msgpack-python==0.4.8 +msgpack-python==0.5.1 netaddr==0.7.19 netifaces==0.10.6 -oslo.config==5.1.0 +oslo.cache==1.28.0 +oslo.concurrency==3.24.0 +oslo.config==5.2.0 oslo.context==2.19.2 +oslo.db==4.33.0 oslo.i18n==3.19.0 -oslo.log==3.35.0 +oslo.log==3.36.0 +oslo.messaging==5.35.0 +oslo.middleware==3.33.0 oslo.policy==1.22.1 -oslo.serialization==2.22.0 -oslo.utils==3.33.0 +oslo.serialization==2.23.0 +oslo.service==1.29.0 +oslo.utils==3.34.0 +Paste==2.0.3 PasteDeploy==1.5.2 pbr==3.0.1 +pep8==1.5.7 +pika==0.11.2 +pika-pool==0.1.3 ply==3.10 positional==1.2.1 +prettytable==0.7.2 +psycopg2==2.7.3.1 pyasn1==0.4.2 pyasn1-modules==0.2.1 pycadf==2.6.0 +pyflakes==0.8.1 pyinotify==0.9.6 pyparsing==2.2.0 +pyperclip==1.6.0 +python-barbicanclient==4.5.2 python-dateutil==2.6.1 +python-editor==1.0.3 python-keystoneclient==3.14.0 +python-memcached==1.58 python-mimeparse==1.6.0 pytz==2017.3 PyYAML==3.12 +repoze.lru==0.7 requests==2.18.4 rfc3986==1.1.0 +Routes==2.4.1 rsa==3.4.2 six==1.11.0 +SQLAlchemy==1.2.0 +sqlalchemy-migrate==0.11.0 +sqlparse==0.2.4 +statsd==3.2.2 stevedore==1.28.0 +Tempita==0.5.2 +tenacity==4.8.0 urllib3==1.22 uWSGI==2.0.15 +vine==1.1.4 WebOb==1.7.4 websocket-client==0.40.0 wrapt==1.10.11 diff --git a/tools/g2/manifests/smoke.json b/tools/g2/manifests/smoke.json index 1a9bb9a4..2863dd54 100644 --- a/tools/g2/manifests/smoke.json +++ b/tools/g2/manifests/smoke.json @@ -1,6 +1,6 @@ { "configuration": [ - "examples/basic", + "examples/complete", "promenade/schemas" ], "stages": [ diff --git a/tools/gate/config-templates/bootstrap-armada-config.yaml b/tools/gate/config-templates/bootstrap-armada-config.yaml index 15e73e09..58039738 100644 --- a/tools/gate/config-templates/bootstrap-armada-config.yaml +++ b/tools/gate/config-templates/bootstrap-armada-config.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: release_prefix: ucp chart_groups: @@ -22,6 +23,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Kubernetes proxy sequenced: true @@ -35,6 +37,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Container networking via Calico sequenced: true @@ -49,6 +52,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Cluster DNS chart_group: @@ -61,6 +65,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: UCP Services chart_group: @@ -73,6 +78,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: description: Kubernetes components chart_group: @@ -85,6 +91,10 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: helm-toolkit + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext data: chart_name: helm-toolkit release: helm-toolkit @@ -107,6 +117,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -161,6 +172,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -403,6 +415,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -465,6 +478,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -542,6 +556,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -640,6 +655,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -709,6 +725,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -768,6 +785,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext substitutions: - src: @@ -968,6 +986,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: chart_name: promenade release: promenade diff --git a/tools/gate/config-templates/genesis-config.yaml b/tools/gate/config-templates/genesis-config.yaml index b132befe..d3101f09 100644 --- a/tools/gate/config-templates/genesis-config.yaml +++ b/tools/gate/config-templates/genesis-config.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: hostname: ${GENESIS_HOSTNAME} ip: ${GENESIS_IP} diff --git a/tools/gate/config-templates/joining-host-config.yaml b/tools/gate/config-templates/joining-host-config.yaml index 79312d18..f7c110e4 100644 --- a/tools/gate/config-templates/joining-host-config.yaml +++ b/tools/gate/config-templates/joining-host-config.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: hostname: ${GENESIS_HOSTNAME} ip: ${GENESIS_IP} @@ -34,6 +35,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: hostname: ${MASTER1_HOSTNAME} ip: ${MASTER1_IP} @@ -62,6 +64,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: hostname: ${MASTER2_HOSTNAME} ip: ${MASTER2_IP} @@ -90,6 +93,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: hostname: ${WORKER_HOSTNAME} ip: ${WORKER_IP} diff --git a/tools/gate/config-templates/site-config.yaml b/tools/gate/config-templates/site-config.yaml index f2d8e19d..373f1489 100644 --- a/tools/gate/config-templates/site-config.yaml +++ b/tools/gate/config-templates/site-config.yaml @@ -6,6 +6,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: dns: cluster_domain: cluster.local @@ -34,6 +35,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: config: insecure-registries: @@ -50,6 +52,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: files: - path: /opt/kubernetes/bin/kubelet @@ -109,6 +112,7 @@ metadata: layeringDefinition: abstract: false layer: site + storagePolicy: cleartext data: arguments: - --cni-bin-dir=/opt/cni/bin