Shipyard updates

This PS makes the following changes:

- makes sure the airskiff-deploy playbook is using 80Gb partition if
  available
- adds available security updates to docker images

Change-Id: Ibcb8fcddb319c3032483fdb73b55b725ea7737d4

.gitignore

@@ -118,3 +118,12 @@ AUTHORS
 
 # vscode
 .vscode/
+
+# tests
+airship-ucp-shipyard.values.yaml
+airflow-webserver.pid
+airflow.cfg
+airflow.db
+latest
+src/bin/shipyard_airflow/shipyard_airflow/config
+src/bin/shipyard_airflow/shipyard_airflow/webserver_config.py

images/airflow/Dockerfile.ubuntu_bionic

@@ -54,7 +54,7 @@ ARG KUBECTL_VERSION=1.24.6
 ENV AIRFLOW_HOME=${AIRFLOW_HOME}
 
 RUN set -ex && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
     apt-get -y install \
     automake \
     ca-certificates \

images/airflow/Dockerfile.ubuntu_focal

@@ -54,7 +54,7 @@ ARG KUBECTL_VERSION=1.26.3
 ENV AIRFLOW_HOME=${AIRFLOW_HOME}
 
 RUN set -ex \
-    && apt-get update \
+    && apt-get update && apt-get upgrade -y \
     && apt-get install -y wget curl \
         apt-transport-https ca-certificates gnupg \
     && echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \

images/shipyard/Dockerfile.ubuntu_bionic

@@ -36,7 +36,7 @@ ARG ctx_base=src/bin
 EXPOSE $PORT
 
 RUN set -ex && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
     apt-get -y install \
     automake \
     ca-certificates \

images/shipyard/Dockerfile.ubuntu_focal

@@ -36,7 +36,7 @@ ARG ctx_base=src/bin
 EXPOSE $PORT
 
 RUN set -ex && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
     apt-get -y install \
     automake \
     ca-certificates \

tools/gate/playbooks/airskiff-deploy.yaml

@@ -23,6 +23,13 @@
 
   tasks:
 
+    - name: Install Packaging python module for airship
+      block:
+        - pip:
+            name: packaging
+            executable: pip3
+      become: True
+
     - name: Clone Required Repositories
       shell: |
         export CLONE_SHIPYARD={{ CLONE_SHIPYARD }}
@@ -39,25 +46,61 @@
 
     - name: Deploy Kubernetes with Minikube
       shell: |
+        set -ex
+        sudo fdisk --list
+        df -h
+        sudo mkdir -p /opt/ext_vol
+        BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep  -E 80G | grep Linux | awk '{print $1}')
+        if ! mount | grep "${BIG_VOLUME}"
+        then
+          sudo mkfs.ext4 "${BIG_VOLUME}"
+          sudo mount "${BIG_VOLUME}" /opt/ext_vol
+          df -h
+        fi
         ./tools/deployment/airskiff/developer/010-deploy-k8s.sh
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
 
+    - name: Setup openstack client
+      shell: |
+        set -ex
+        ./tools/deployment/airskiff/developer/020-setup-client.sh
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
     - name: Build Shipyard and Airflow with submitted changes
       shell: |
         set -ex
         export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
         make images
-        if test "${DISTRO}" = 'ubuntu_bionic'
+        docker system prune --force
-        then
-          # this trick is needed to use bionic image instead of focal in airskiff deployment test
-          docker tag quay.io/airshipit/airflow:latest-ubuntu_bionic quay.io/airshipit/airflow:latest-ubuntu_focal
-          docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic quay.io/airshipit/shipyard:latest-ubuntu_focal
-        fi
       args:
         chdir: "{{ zuul.project.src_dir }}"
       become: yes
 
+    - name: Use locally built images in manifests
+      shell: |
+        set -ex
+        export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
+        docker rm registry --force || true
+        docker run -d -p 5000:5000 --restart=always --name registry registry:2
+        if test "${DISTRO}" = 'ubuntu_bionic'
+        then
+          docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic localhost:5000/shipyard:latest-ubuntu_bionic
+          docker push localhost:5000/shipyard:latest-ubuntu_bionic
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_bionic#g" ./site/airskiff/software/config/versions.yaml
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_bionic#g" ./global/software/config/versions.yaml
+        else
+          docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal localhost:5000/shipyard:latest-ubuntu_focal
+          docker push localhost:5000/shipyard:latest-ubuntu_focal
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml
+          sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./global/software/config/versions.yaml
+        fi
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
     - name: Build all charts locally
       shell: |
         set -ex
@@ -66,6 +109,18 @@
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
       become: yes
 
+    - name: Start artifactory
+      shell: |
+        set -ex
+        # start http server with artifacts
+        docker rm artifacts --force || true
+        docker run --name artifacts -p 8282:80 -v $(pwd)/../artifacts:/usr/share/nginx/html -d nginx
+        sleep 10
+        curl --verbose -I http://control-plane.minikube.internal:8282/memcached.tgz
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
     - name: Deploy Airship components using Armada
       shell: |
         mkdir -p ~/.kube

tools/gate/playbooks/docker-image-build.yaml

@@ -4,7 +4,6 @@
     - ensure-docker
     - ensure-python
     - ensure-pip
-    - ensure-tox
 
   tasks:
     - include_vars: vars.yaml
@@ -43,6 +42,14 @@
             executable: pip3
       become: True
 
+    - name: Install tox python module for ansible docker login
+      block:
+        - pip:
+            name: tox
+            version: 3.28.0
+            executable: pip3
+      become: True
+
     - name: Make images - verbosive
       when: not publish
       shell: |