Add complete support for operator-based bootstrap

* operator logs is now streaming to pipeline and to pod
 * printing status of armada chart objects
 * adjust armada container cmd parameters to support both
   golang and python based images

Signed-off-by: Ruslan Aliev <raliev@mirantis.com>
Change-Id: I6d8629a48c1b862db937ddc3cd68792220388b19

promenade/templates/roles/genesis/etc/kubernetes/manifests/bootstrap-armada.yaml

@@ -21,6 +21,11 @@ spec:
         - |-
           set -x
 
+          if [ ! -e "${ARMADA_LOGFILE}" ]; then
+            touch "${ARMADA_LOGFILE}"
+          fi
+          tail -f "${ARMADA_LOGFILE}" &
+
           declare -i attempt=1
 
           while true; do
@@ -28,9 +33,6 @@ spec:
               if armada \
                       apply \
                       --target-manifest {{ config.get_path('Genesis:armada.target_manifest', 'cluster-bootstrap') }} \
-{%- if config['Genesis:enable_operator'] is sameas true %}
-                      --enable-operator \
-{%- endif %}
                       $([[ $attempt -le $ARMADA_METRICS_MAX_ATTEMPTS ]] && echo --metrics-output "${ARMADA_METRICS_OUTPUT_DIR}/armada-bootstrap-${attempt}.prom") \
                       /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
                   break
@@ -72,14 +74,25 @@ spec:
         - name: metrics
           mountPath: /tmp/metrics
 {%- if config['Genesis:enable_operator'] is sameas true %}
-    - args:
-      - --health-probe-bind-address=:8081
-      - --metrics-bind-address=127.0.0.1:8080
-      - --leader-elect
-      - --leader-elect-namespace=ucp
-      command:
-      - /manager
+    - command:
+        - /bin/sh
+        - -c
+        - |-
+          set -x
+
+          if [ ! -e "${ARMADA_OPERATOR_LOGFILE}" ]; then
+            touch "${ARMADA_OPERATOR_LOGFILE}"
+          fi
+          tail -f "${ARMADA_OPERATOR_LOGFILE}" &
+
+          /manager \
+            --health-probe-bind-address=:8081 \
+            --metrics-bind-address=127.0.0.1:8080 \
+            --leader-elect \
+            --leader-elect-namespace=ucp 2>&1 | tee -a "${ARMADA_OPERATOR_LOGFILE}"
       env:
+      - name: ARMADA_OPERATOR_LOGFILE
+        value: /tmp/log/bootstrap-armada-operator.log
       - name: KUBECONFIG
         value: /root/.kube/config
       image: {{ config['Genesis:images.armada-operator'] }}
@@ -101,6 +114,8 @@ spec:
       volumeMounts:
         - name: auth
           mountPath: /root/.kube
+        - name: log
+          mountPath: /tmp/log
 {%- endif %}
     - name: monitor
       image: {{ config['HostSystem:images.monitoring_image'] }}

promenade/templates/scripts/genesis.sh

@@ -7,6 +7,10 @@
 mkdir -p /var/log/armada
 touch /var/log/armada/bootstrap-armada.log
 chmod 777 /var/log/armada/bootstrap-armada.log
+{%- if config['Genesis:enable_operator'] is sameas true %}
+touch /var/log/armada/bootstrap-armada-operator.log
+chmod 777 /var/log/armada/bootstrap-armada-operator.log
+{%- endif %}
 
 {% set metrics_output_dir = config.get_path('Genesis:armada.metrics.output_dir', '/var/log/node-exporter-textfiles') %}
 install -d -m 755 {{ metrics_output_dir }}
@@ -33,16 +37,27 @@ log
 log === Deploying bootstrap manifest via Armada ===
 set -x
 
+{%- if config['Genesis:enable_operator'] is sameas true %}
+while [[ ! -e /var/log/armada/bootstrap-armada.log || ! -e /var/log/armada/bootstrap-armada-operator.log ]]; do
+{%- else %}
 while [[ ! -e /var/log/armada/bootstrap-armada.log ]]; do
+{%- endif %}
     sleep 5
 done
 tail -f /var/log/armada/bootstrap-armada.log &
+{%- if config['Genesis:enable_operator'] is sameas true %}
+tail -f /var/log/armada/bootstrap-armada-operator.log &
+{%- endif %}
 
 set +x
 while true; do
     if [[ -e /etc/kubernetes/manifests/bootstrap-armada.yaml ]]; then
         sleep 30
         kubectl get pods --all-namespaces || echo "Could not get current pod status."
+        {%- if config['Genesis:enable_operator'] is sameas true %}
+        kubectl get armadacharts \
+         -o=custom-columns='NAMESPACE:metadata.namespace,NAME:metadata.name,READY:status.conditions[?(@.type=="Ready")].status,MESSAGE:status.conditions[?(@.type=="Ready")].message' -A || echo "Could not get current armada chart status."
+        {%- endif %}
     else
         log Armada bootstrap manifest deployed
         break
@@ -52,6 +67,9 @@ set -x
 
 # Terminate background job (tear down exit trap?)
 kill %1
+{%- if config['Genesis:enable_operator'] is sameas true %}
+kill %2
+{%- endif %}
 
 set +x
 log