diff --git a/.gitignore b/.gitignore index 740ab8e3..079a1660 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,12 @@ AUTHORS # vscode .vscode/ + +# tests +airship-ucp-shipyard.values.yaml +airflow-webserver.pid +airflow.cfg +airflow.db +latest +src/bin/shipyard_airflow/shipyard_airflow/config +src/bin/shipyard_airflow/shipyard_airflow/webserver_config.py \ No newline at end of file diff --git a/images/airflow/Dockerfile.ubuntu_bionic b/images/airflow/Dockerfile.ubuntu_bionic index b4134a21..b276e82f 100644 --- a/images/airflow/Dockerfile.ubuntu_bionic +++ b/images/airflow/Dockerfile.ubuntu_bionic @@ -54,7 +54,7 @@ ARG KUBECTL_VERSION=1.24.6 ENV AIRFLOW_HOME=${AIRFLOW_HOME} RUN set -ex && \ - apt-get -qq update && \ + apt-get update && apt-get upgrade -y && \ apt-get -y install \ automake \ ca-certificates \ diff --git a/images/airflow/Dockerfile.ubuntu_focal b/images/airflow/Dockerfile.ubuntu_focal index 42415b9f..dc64c5c3 100644 --- a/images/airflow/Dockerfile.ubuntu_focal +++ b/images/airflow/Dockerfile.ubuntu_focal @@ -54,7 +54,7 @@ ARG KUBECTL_VERSION=1.26.3 ENV AIRFLOW_HOME=${AIRFLOW_HOME} RUN set -ex \ - && apt-get update \ + && apt-get update && apt-get upgrade -y \ && apt-get install -y wget curl \ apt-transport-https ca-certificates gnupg \ && echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ diff --git a/images/shipyard/Dockerfile.ubuntu_bionic b/images/shipyard/Dockerfile.ubuntu_bionic index 77443cd3..49baadaa 100644 --- a/images/shipyard/Dockerfile.ubuntu_bionic +++ b/images/shipyard/Dockerfile.ubuntu_bionic @@ -36,7 +36,7 @@ ARG ctx_base=src/bin EXPOSE $PORT RUN set -ex && \ - apt-get -qq update && \ + apt-get update && apt-get upgrade -y && \ apt-get -y install \ automake \ ca-certificates \ diff --git a/images/shipyard/Dockerfile.ubuntu_focal b/images/shipyard/Dockerfile.ubuntu_focal index b62f6fec..49d377d3 100644 --- a/images/shipyard/Dockerfile.ubuntu_focal +++ b/images/shipyard/Dockerfile.ubuntu_focal @@ -36,7 +36,7 @@ ARG ctx_base=src/bin EXPOSE $PORT RUN set -ex && \ - apt-get -qq update && \ + apt-get update && apt-get upgrade -y && \ apt-get -y install \ automake \ ca-certificates \ diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index c4d7c3bd..f4158e43 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -23,6 +23,13 @@ tasks: + - name: Install Packaging python module for airship + block: + - pip: + name: packaging + executable: pip3 + become: True + - name: Clone Required Repositories shell: | export CLONE_SHIPYARD={{ CLONE_SHIPYARD }} @@ -39,25 +46,61 @@ - name: Deploy Kubernetes with Minikube shell: | + set -ex + sudo fdisk --list + df -h + sudo mkdir -p /opt/ext_vol + BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep -E 80G | grep Linux | awk '{print $1}') + if ! mount | grep "${BIG_VOLUME}" + then + sudo mkfs.ext4 "${BIG_VOLUME}" + sudo mount "${BIG_VOLUME}" /opt/ext_vol + df -h + fi ./tools/deployment/airskiff/developer/010-deploy-k8s.sh args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + - name: Setup openstack client + shell: | + set -ex + ./tools/deployment/airskiff/developer/020-setup-client.sh + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + - name: Build Shipyard and Airflow with submitted changes shell: | set -ex export DISTRO={{ SHIPYARD_IMAGE_DISTRO }} make images - if test "${DISTRO}" = 'ubuntu_bionic' - then - # this trick is needed to use bionic image instead of focal in airskiff deployment test - docker tag quay.io/airshipit/airflow:latest-ubuntu_bionic quay.io/airshipit/airflow:latest-ubuntu_focal - docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic quay.io/airshipit/shipyard:latest-ubuntu_focal - fi + docker system prune --force args: chdir: "{{ zuul.project.src_dir }}" become: yes + - name: Use locally built images in manifests + shell: | + set -ex + export DISTRO={{ SHIPYARD_IMAGE_DISTRO }} + docker rm registry --force || true + docker run -d -p 5000:5000 --restart=always --name registry registry:2 + if test "${DISTRO}" = 'ubuntu_bionic' + then + docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic localhost:5000/shipyard:latest-ubuntu_bionic + docker push localhost:5000/shipyard:latest-ubuntu_bionic + sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_bionic#g" ./site/airskiff/software/config/versions.yaml + sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_bionic#g" ./global/software/config/versions.yaml + else + docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal localhost:5000/shipyard:latest-ubuntu_focal + docker push localhost:5000/shipyard:latest-ubuntu_focal + sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml + sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./global/software/config/versions.yaml + fi + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + - name: Build all charts locally shell: | set -ex @@ -66,6 +109,18 @@ chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" become: yes + - name: Start artifactory + shell: | + set -ex + # start http server with artifacts + docker rm artifacts --force || true + docker run --name artifacts -p 8282:80 -v $(pwd)/../artifacts:/usr/share/nginx/html -d nginx + sleep 10 + curl --verbose -I http://control-plane.minikube.internal:8282/memcached.tgz + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + - name: Deploy Airship components using Armada shell: | mkdir -p ~/.kube diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml index 32ccc3b8..fd6c8662 100644 --- a/tools/gate/playbooks/docker-image-build.yaml +++ b/tools/gate/playbooks/docker-image-build.yaml @@ -4,7 +4,6 @@ - ensure-docker - ensure-python - ensure-pip - - ensure-tox tasks: - include_vars: vars.yaml @@ -43,6 +42,14 @@ executable: pip3 become: True + - name: Install tox python module for ansible docker login + block: + - pip: + name: tox + version: 3.28.0 + executable: pip3 + become: True + - name: Make images - verbosive when: not publish shell: |