Use helm-toolkit for readiness probes

Use helm toolkit for airflow worker readiness probes.

Also PS makes the following changes:

- makes sure the airskiff-deploy playbook is using 80Gb partition if
  available
- adds available security updates to docker images

Change-Id: I6cf79cb02e1f7752634943c00113cc0560d69f8e

.gitignore

@@ -118,3 +118,12 @@ AUTHORS
 
 # vscode
 .vscode/
+
+# tests
+airship-ucp-shipyard.values.yaml
+airflow-webserver.pid
+airflow.cfg
+airflow.db
+latest
+src/bin/shipyard_airflow/shipyard_airflow/config
+src/bin/shipyard_airflow/shipyard_airflow/webserver_config.py

charts/shipyard/Chart.yaml

@@ -16,7 +16,7 @@
 apiVersion: v1
 description: A Helm chart for Shipyard and Airflow
 name: shipyard
-version: 0.1.1
+version: 0.1.2
 keywords:
 - shipyard
 - airflow

charts/shipyard/templates/statefulset-airflow-worker.yaml

@@ -12,6 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{{- define "airflowLogrotateReadinessProbeTemplate" }}
+exec:
+  command:
+    - sh
+    - -c
+    - |-
+    set -ex
+    ps -ef | grep 'sleep 3600' | grep -v 'grep'
+{{- end }}
+
+{{- define "airflowWorkerReadinessProbeTemplate" }}
+tcpSocket:
+  port: {{ .Values.network.airflow.worker.port }}
+{{- end }}
+
 {{- if .Values.manifests.statefulset_airflow_worker }}
 {{- $envAll := . }}
 
@@ -143,6 +158,7 @@ spec:
           imagePullPolicy: {{ .Values.images.pull_policy }}
 {{ tuple $envAll $envAll.Values.pod.resources.airflow.worker | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "airflow_worker" "container" "airflow_worker" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "component" "worker" "container" "airflow_worker" "type" "readiness" "probeTemplate" (include "airflowWorkerReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           env:
           - name: AIRFLOW_CONN_AIRFLOWS_OWN_DB
             valueFrom:
@@ -152,9 +168,6 @@ spec:
           ports:
             - containerPort: {{ tuple "airflow_worker" "internal" "airflow_worker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           args: ["worker"]
-          readinessProbe:
-            tcpSocket:
-              port: {{ .Values.network.airflow.worker.port }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
@@ -177,6 +190,7 @@ spec:
           imagePullPolicy: {{ .Values.images.pull_policy }}
 {{ tuple $envAll $envAll.Values.pod.resources.airflow.logrotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "airflow_worker" "container" "airflow_logrotate" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" $envAll "component" "worker" "container" "airflow_logrotate" "type" "readiness" "probeTemplate" (include "airflowLogrotateReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
           env:
           - name: PERCENT_MAX_LOG_FS_USAGE
             value: {{ .Values.logrotate.percent_max_log_fs_usage | quote }}
@@ -186,16 +200,6 @@ spec:
             value: {{ .Values.conf.airflow.core.base_log_folder }}
           - name: LOGROTATE_ENABLED
             value: "enabled"
-          readinessProbe:
-            exec:
-              command:
-              - sh
-              - -c
-              - |-
-                set -ex
-                ps -ef | grep 'sleep 3600' | grep -v 'grep'
-            initialDelaySeconds: 15
-            periodSeconds: 15
           volumeMounts:
             - name: airflow-logs
               mountPath: {{ .Values.conf.airflow.core.base_log_folder }}

charts/shipyard/values.yaml

@@ -863,6 +863,18 @@ pod:
     airflow:
       worker: 2
       scheduler: 2
+  probes:
+    worker:
+      airflow_worker:
+        readiness:
+          enabled: true
+      airflow_logrotate:
+        readiness:
+          enabled: true
+          params:
+            initialDelaySeconds: 15
+            periodSeconds: 15
+            timeoutSeconds: 10
   lifecycle:
     upgrades:
       deployments:

images/airflow/Dockerfile.ubuntu_bionic

@@ -54,7 +54,7 @@ ARG KUBECTL_VERSION=1.24.6
 ENV AIRFLOW_HOME=${AIRFLOW_HOME}
 
 RUN set -ex && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
     apt-get -y install \
     automake \
     ca-certificates \

images/airflow/Dockerfile.ubuntu_focal

@@ -54,7 +54,7 @@ ARG KUBECTL_VERSION=1.26.3
 ENV AIRFLOW_HOME=${AIRFLOW_HOME}
 
 RUN set -ex \
-    && apt-get update \
+    && apt-get update && apt-get upgrade -y \
     && apt-get install -y wget curl \
         apt-transport-https ca-certificates gnupg \
     && echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \

images/shipyard/Dockerfile.ubuntu_bionic

@@ -36,7 +36,7 @@ ARG ctx_base=src/bin
 EXPOSE $PORT
 
 RUN set -ex && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
     apt-get -y install \
     automake \
     ca-certificates \

images/shipyard/Dockerfile.ubuntu_focal

@@ -36,7 +36,7 @@ ARG ctx_base=src/bin
 EXPOSE $PORT
 
 RUN set -ex && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
     apt-get -y install \
     automake \
     ca-certificates \

tools/gate/playbooks/airskiff-deploy.yaml

@@ -23,6 +23,13 @@
 
   tasks:
 
+    - name: Install Packaging python module for airship
+      block:
+        - pip:
+            name: packaging
+            executable: pip3
+      become: True
+
     - name: Clone Required Repositories
       shell: |
         export CLONE_SHIPYARD={{ CLONE_SHIPYARD }}
@@ -39,25 +46,61 @@
 
     - name: Deploy Kubernetes with Minikube
       shell: |
+        set -ex
+        sudo fdisk --list
+        df -h
+        sudo mkdir -p /opt/ext_vol
+        BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep  -E 80G | grep Linux | awk '{print $1}')
+        if ! mount | grep "${BIG_VOLUME}"
+        then
+          sudo mkfs.ext4 "${BIG_VOLUME}"
+          sudo mount "${BIG_VOLUME}" /opt/ext_vol
+          df -h
+        fi
         ./tools/deployment/airskiff/developer/010-deploy-k8s.sh
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
 
+    - name: Setup openstack client
+      shell: |
+        set -ex
+        ./tools/deployment/airskiff/developer/020-setup-client.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
     - name: Build Shipyard and Airflow with submitted changes
       shell: |
         set -ex
         export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
         make images
-        if test "${DISTRO}" = 'ubuntu_bionic'
-        then
-          # this trick is needed to use bionic image instead of focal in airskiff deployment test
-          docker tag quay.io/airshipit/airflow:latest-ubuntu_bionic quay.io/airshipit/airflow:latest-ubuntu_focal
-          docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic quay.io/airshipit/shipyard:latest-ubuntu_focal
-        fi
+        docker system prune --force
       args:
         chdir: "{{ zuul.project.src_dir }}"
       become: yes
 
+    - name: Use locally built images in manifests
+      shell: |
+        set -ex
+        export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
+        docker rm registry --force || true
+        docker run -d -p 5000:5000 --restart=always --name registry registry:2
+        if test "${DISTRO}" = 'ubuntu_bionic'
+        then
+          docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic localhost:5000/shipyard:latest-ubuntu_bionic
+          docker push localhost:5000/shipyard:latest-ubuntu_bionic
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_bionic#g" ./site/airskiff/software/config/versions.yaml
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_bionic#g" ./global/software/config/versions.yaml
+        else
+          docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal localhost:5000/shipyard:latest-ubuntu_focal
+          docker push localhost:5000/shipyard:latest-ubuntu_focal
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./global/software/config/versions.yaml
+        fi
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
     - name: Build all charts locally
       shell: |
         set -ex
@@ -66,6 +109,18 @@
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
       become: yes
 
+    - name: Start artifactory
+      shell: |
+        set -ex
+        # start http server with artifacts
+        docker rm artifacts --force || true
+        docker run --name artifacts -p 8282:80 -v $(pwd)/../artifacts:/usr/share/nginx/html -d nginx
+        sleep 10
+        curl --verbose -I http://control-plane.minikube.internal:8282/memcached.tgz
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
     - name: Deploy Airship components using Armada
       shell: |
         mkdir -p ~/.kube

tools/gate/playbooks/docker-image-build.yaml

@@ -4,7 +4,6 @@
     - ensure-docker
     - ensure-python
     - ensure-pip
-    - ensure-tox
 
   tasks:
     - include_vars: vars.yaml
@@ -43,6 +42,14 @@
             executable: pip3
       become: True
 
+    - name: Install tox python module for ansible docker login
+      block:
+        - pip:
+            name: tox
+            version: 3.28.0
+            executable: pip3
+      become: True
+
     - name: Make images - verbosive
       when: not publish
       shell: |