Update apiserver for admission controller

Update the apiserver's command prefix to match between the apiserver
chart and the genesis api server.

Includes an optimization of virsh commands to parallelize and wait for
cloud init to complete (requires Ubuntu 16.04.4)

Change-Id: I793d210e15c3c13e8b662bd2459ed8bcba30d434

deployment_files/global/v1.0demo/profiles/genesis.yaml

@@ -91,8 +91,11 @@ data:
       - --service-cluster-ip-range=SERVICE_CIDR
       - --service-node-port-range=SERVICE_NODE_PORT_RANGE
       - --authorization-mode=Node,RBAC
-      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
       - --endpoint-reconciler-type=lease
+      # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
+      - --repair-malformed-updates=false
+
   armada:
     target_manifest: cluster-bootstrap
   labels:

deployment_files/global/v1.0demo/software/charts/kubernetes/core/apiserver.yaml

@@ -97,6 +97,22 @@ metadata:
       dest:
         path: .values.secrets.service_account.public_key
 
+     # Command prefix
+    - src:
+        schema: pegleg/CommonAddresses/v1
+        name: common-addresses
+        path: .kubernetes.service_cidr
+      dest:
+        path: .values.command_prefix[1]
+        pattern: SERVICE_CIDR
+    - src:
+        schema: pegleg/CommonAddresses/v1
+        name: common-addresses
+        path: .kubernetes.service_node_port_range
+      dest:
+        path: .values.command_prefix[2]
+        pattern: SERVICE_NODE_PORT_RANGE
+
 data:
   chart_name: apiserver
   release: kubernetes-apiserver
@@ -115,9 +131,20 @@ data:
           labels:
             release_group: airship-kubernetes-apiserver
   values:
+
     apiserver:
       etcd:
         endpoints: https://127.0.0.1:2378
+    command_prefix:
+      - /apiserver
+      - --service-cluster-ip-range=SERVICE_CIDR
+      - --service-node-port-range=SERVICE_NODE_PORT_RANGE
+      - --authorization-mode=Node,RBAC
+      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
+      - --endpoint-reconciler-type=lease
+      # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
+      - --repair-malformed-updates=false
+
   dependencies:
     - kubernetes-apiserver-htk
 ---

deployment_files/global/v1.0demo/software/config/versions.yaml

@@ -19,18 +19,18 @@ data:
         type: git
       apiserver-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       calico:
         calico:
           location: https://git.openstack.org/openstack/openstack-helm-infra
-          reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+          reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
           subpath: calico
           type: git
         calico-htk:
           location: https://git.openstack.org/openstack/openstack-helm-infra
-          reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+          reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
           subpath: helm-toolkit
           type: git
         etcd:
@@ -40,7 +40,7 @@ data:
           type: git
         etcd-htk:
           location: https://git.openstack.org/openstack/openstack-helm-infra
-          reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+          reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
           subpath: helm-toolkit
           type: git
       controller-manager:
@@ -50,7 +50,7 @@ data:
         type: git
       controller-manager-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       coredns:
@@ -60,7 +60,7 @@ data:
         type: git
       coredns-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       etcd:
@@ -70,7 +70,7 @@ data:
         type: git
       etcd-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       haproxy:
@@ -80,17 +80,17 @@ data:
         type: git
       haproxy-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       ingress:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: ingress
         type: git
       ingress-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       proxy:
@@ -100,7 +100,7 @@ data:
         type: git
       proxy-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       scheduler:
@@ -110,7 +110,7 @@ data:
         type: git
       scheduler-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
     ucp:
@@ -118,12 +118,12 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: nfs-provisioner
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       nfs_htk:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       armada:
         location: https://git.openstack.org/openstack/airship-armada
         reference: 32baf9851e9cd29f4b8670e73a9409a0e7a59e35
@@ -131,7 +131,7 @@ data:
         type: git
       armada-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       barbican:
@@ -141,7 +141,7 @@ data:
         type: git
       barbican-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       #TODO Update to split ceph charts
@@ -162,7 +162,7 @@ data:
         type: git
       deckhand-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       divingbell:
@@ -172,27 +172,27 @@ data:
         type: git
       divingbell-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       drydock:
         location: https://git.openstack.org/openstack/airship-drydock
-        reference: 756a063c3074ddd847af1dd02c9b576b541cedbd
+        reference: 3470f89519d018c572905566891bafc8a3c51c3b
         subpath: charts/drydock
         type: git
       drydock-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       ingress:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: ingress
         type: git
       ingress-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       keystone:
@@ -202,7 +202,7 @@ data:
         type: git
       keystone-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       maas:
@@ -212,37 +212,37 @@ data:
         type: git
       maas-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       mariadb:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: mariadb
         type: git
       mariadb-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       memcached:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: memcached
         type: git
       memcached-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       postgresql:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: postgresql
         type: git
       postgresql-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       promenade:
@@ -252,17 +252,17 @@ data:
         type: git
       promenade-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       rabbitmq:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: rabbitmq
         type: git
       rabbitmq-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       shipyard:
@@ -272,7 +272,7 @@ data:
         type: git
       shipyard-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
       tiller:
@@ -282,7 +282,7 @@ data:
         type: git
       tiller-htk:
         location: https://git.openstack.org/openstack/openstack-helm-infra
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         subpath: helm-toolkit
         type: git
     openstack:
@@ -295,7 +295,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       heat:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -305,7 +305,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       horizon:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -315,17 +315,17 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       ingress:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: ingress
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       ingress-htk:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       keystone:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -335,7 +335,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       libvirt:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -345,7 +345,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       mariadb:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -355,17 +355,17 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       memcached:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: memcached
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       memcached-htk:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       neutron:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -375,7 +375,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       nova:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -385,7 +385,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       openvswitch:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -395,17 +395,17 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       rabbitmq:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: rabbitmq
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       rabbitmq-htk:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
 
   files:
     kubelet: https://dl.k8s.io/v1.10.2/kubernetes-node-linux-amd64.tar.gz
@@ -470,9 +470,9 @@ data:
         divingbell: docker.io/ubuntu:16.04
       drydock:
         dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
-        drydock: quay.io/airshipit/drydock:04ab7a6a90d2b0aadeabed804df31c9927538f91
+        drydock: quay.io/airshipit/drydock:3470f89519d018c572905566891bafc8a3c51c3b
         drydock_db_init: docker.io/postgres:9.5
-        drydock_db_sync: quay.io/airshipit/drydock:04ab7a6a90d2b0aadeabed804df31c9927538f91
+        drydock_db_sync: quay.io/airshipit/drydock:3470f89519d018c572905566891bafc8a3c51c3b
         ks_endpoints: docker.io/openstackhelm/heat:ocata
         ks_service: docker.io/openstackhelm/heat:ocata
         ks_user: docker.io/openstackhelm/heat:ocata

deployment_files/global/v1.0dev/profiles/genesis.yaml

@@ -91,8 +91,11 @@ data:
       - --service-cluster-ip-range=SERVICE_CIDR
       - --service-node-port-range=SERVICE_NODE_PORT_RANGE
       - --authorization-mode=Node,RBAC
-      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
       - --endpoint-reconciler-type=lease
+      # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
+      - --repair-malformed-updates=false
+
   armada:
     target_manifest: cluster-bootstrap
   labels:
@@ -109,11 +112,11 @@ data:
       - openvswitch=enabled
       - promenade-genesis=enabled
       - ucp-control-plane=enabled
-      - ceph-mgr=enabled
       - ceph-mon=enabled
       - ceph-mds=enabled
-      - ceph-rgw=enabled
       - ceph-osd=enabled
+      - ceph-rgw=enabled
+      - ceph-mgr=enabled
   files:
     - path: /var/lib/anchor/calico-etcd-bootstrap
       content: "# placeholder for triggering calico etcd bootstrapping\n# this file will be deleted"

deployment_files/global/v1.0dev/software/charts/kubernetes/core/apiserver.yaml

@@ -97,6 +97,22 @@ metadata:
       dest:
         path: .values.secrets.service_account.public_key
 
+     # Command prefix
+    - src:
+        schema: pegleg/CommonAddresses/v1
+        name: common-addresses
+        path: .kubernetes.service_cidr
+      dest:
+        path: .values.command_prefix[1]
+        pattern: SERVICE_CIDR
+    - src:
+        schema: pegleg/CommonAddresses/v1
+        name: common-addresses
+        path: .kubernetes.service_node_port_range
+      dest:
+        path: .values.command_prefix[2]
+        pattern: SERVICE_NODE_PORT_RANGE
+
 data:
   chart_name: apiserver
   release: kubernetes-apiserver
@@ -115,9 +131,20 @@ data:
           labels:
             release_group: airship-kubernetes-apiserver
   values:
+
     apiserver:
       etcd:
         endpoints: https://127.0.0.1:2378
+    command_prefix:
+      - /apiserver
+      - --service-cluster-ip-range=SERVICE_CIDR
+      - --service-node-port-range=SERVICE_NODE_PORT_RANGE
+      - --authorization-mode=Node,RBAC
+      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
+      - --endpoint-reconciler-type=lease
+      # NOTE(mark-burnett): This flag is removed in Kubernetes 1.11
+      - --repair-malformed-updates=false
+
   dependencies:
     - kubernetes-apiserver-htk
 ---

deployment_files/global/v1.0dev/software/config/versions.yaml

@@ -22,17 +22,17 @@ data:
           type: git
           location: https://git.openstack.org/openstack/openstack-helm-infra
           subpath: helm-toolkit
-          reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+          reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         calico:
           type: git
           location: https://git.openstack.org/openstack/openstack-helm-infra
           subpath: calico
-          reference: 26e1b9cde63a3d5a55417d25f05cf8787b03256d
+          reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
         calico-htk:
           type: git
           location: https://git.openstack.org/openstack/openstack-helm-infra
           subpath: helm-toolkit
-          reference: 26e1b9cde63a3d5a55417d25f05cf8787b03256d
+          reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       apiserver:
         type: git
         location: https://git.openstack.org/openstack/airship-promenade
@@ -42,7 +42,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       controller-manager:
         type: git
         location: https://git.openstack.org/openstack/airship-promenade
@@ -52,7 +52,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       coredns:
         type: git
         location: https://git.openstack.org/openstack/airship-promenade
@@ -62,7 +62,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       haproxy:
         type: git
         location: https://git.openstack.org/openstack/airship-promenade
@@ -72,7 +72,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       etcd:
         type: git
         location: https://git.openstack.org/openstack/airship-promenade
@@ -82,17 +82,17 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       ingress:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: ingress
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       ingress-htk:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       proxy:
         type: git
         location: https://git.openstack.org/openstack/airship-promenade
@@ -102,7 +102,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       scheduler:
         type: git
         location: https://git.openstack.org/openstack/airship-promenade
@@ -112,18 +112,18 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
     ucp:
       nfs_provisioner:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: nfs-provisioner
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       nfs_htk:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       armada:
         type: git
         location: https://git.openstack.org/openstack/airship-armada
@@ -133,7 +133,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       barbican:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -143,7 +143,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       #TODO Update to split ceph charts
       ceph:
         type: git
@@ -165,7 +165,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       divingbell:
         type: git
         location: https://git.openstack.org/openstack/airship-divingbell
@@ -175,27 +175,27 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       drydock:
         type: git
         location: https://git.openstack.org/openstack/airship-drydock
         subpath: charts/drydock
-        reference: master
+        reference: 3470f89519d018c572905566891bafc8a3c51c3b
       drydock-htk:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: bc1afb87d7aa529a4ed5321d889cdfe2f1af8a44
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       ingress:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: ingress
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       ingress-htk:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       postgresql:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -205,7 +205,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       promenade:
         type: git
         location: https://git.openstack.org/openstack/airship-promenade
@@ -215,7 +215,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       keystone:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -225,7 +225,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       maas:
         type: git
         location: https://git.openstack.org/openstack/airship-maas
@@ -235,7 +235,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       mariadb:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm
@@ -245,27 +245,27 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       memcached:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: memcached
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       memcached-htk:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       rabbitmq:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: rabbitmq
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       rabbitmq-htk:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
       shipyard:
         type: git
         location: https://git.openstack.org/openstack/airship-shipyard
@@ -285,7 +285,7 @@ data:
         type: git
         location: https://git.openstack.org/openstack/openstack-helm-infra
         subpath: helm-toolkit
-        reference: 6ef48d37060e81cc6ffc283644bcd2df6c7ef80e
+        reference: 5d356f9265b337b75f605dee839faa8cd0ed3ab2
 
   files:
     kubelet: https://dl.k8s.io/v1.10.2/kubernetes-node-linux-amd64.tar.gz
@@ -327,13 +327,13 @@ data:
         ks_user: docker.io/openstackhelm/heat:ocata
         barbican_api: docker.io/openstackhelm/barbican:ocata
       drydock:
-        drydock: quay.io/airshipit/drydock:a41acdd7607ff5281d3df3ea435e15d5ff5c3f99
+        drydock: quay.io/airshipit/drydock:3470f89519d018c572905566891bafc8a3c51c3b
         dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
         ks_user: docker.io/openstackhelm/heat:ocata
         ks_service: docker.io/openstackhelm/heat:ocata
         ks_endpoints: docker.io/openstackhelm/heat:ocata
         drydock_db_init: docker.io/postgres:9.5
-        drydock_db_sync: quay.io/airshipit/drydock:a41acdd7607ff5281d3df3ea435e15d5ff5c3f99
+        drydock_db_sync: quay.io/airshipit/drydock:3470f89519d018c572905566891bafc8a3c51c3b
       shipyard:
         airflow: quay.io/airshipit/airflow:0a14e7e80fb7b7bd41b29e9690f231229c1e95ca
         shipyard: quay.io/airshipit/shipyard:0a14e7e80fb7b7bd41b29e9690f231229c1e95ca

tools/multi_nodes_gate/airship_gate/lib/virsh.sh

@@ -153,7 +153,8 @@ vm_create() {
             --disk "vol=${VIRSH_POOL}/cloud-init-${NAME}.iso,device=cdrom" &>> "${LOG_FILE}"
 
         ssh_wait "${NAME}"
-        ssh_cmd "${NAME}" sync
+	ssh_cmd "${NAME}" cloud-init status --wait
+	ssh_cmd "${NAME}" sync
 
     else
         log Creating VM "${NAME}"
@@ -172,18 +173,24 @@ vm_create() {
     fi
 }
 
+vm_create_validate() {
+    NAME=${1}
+    vm_create "${name}"
+    if [[ "$(config_vm_bootstrap ${name})" == "true" ]]
+    then
+      vm_validate "${name}"
+    fi
+}
+
 vm_create_all() {
     log Starting all VMs
 
     VM_NAMES=($(config_vm_names))
     for name in ${VM_NAMES[*]}
     do
-      vm_create "${name}"
+      vm_create_validate "${name}" &
-      if [[ "$(config_vm_bootstrap ${name})" == "true" ]]
-      then
-        vm_validate "${name}"
-      fi
     done
+    wait
 }
 
 vm_start() {