From 8ce937a9f7ac94d51d4d946b6c2288975c53853a Mon Sep 17 00:00:00 2001 From: "Wahlstedt, Walter (ww229g)" Date: Mon, 19 Dec 2022 16:02:39 -0500 Subject: [PATCH] updates for focal add focal dockerfile update zuul jobs for focal update tox for tox4 changes update all requirements to latest and match deckhand update cfssl from R1.2 to v1.6.3 fixed local gates for focal updated examples promenade manifests to run on focal Change-Id: I2af4043784766d36588c6f738053ad66e7b89a90 --- Makefile | 7 +- charts/promenade/Chart.yaml | 1 + doc/requirements.txt | 2 +- doc/source/conf.py | 4 +- doc/source/configuration/host-system.rst | 2 +- examples/basic/HostSystem.yaml | 12 +- examples/basic/Kubelet.yaml | 2 + examples/basic/armada-resources.yaml | 14 +- examples/complete/HostSystem.yaml | 49 +++- examples/complete/Kubelet.yaml | 7 +- examples/complete/armada-resources.yaml | 12 +- examples/containerd/armada-resources.yaml | 6 +- examples/gate/HostSystem.yaml | 10 +- examples/gate/Kubelet.yaml | 5 +- examples/gate/armada-resources.yaml | 8 +- images/promenade/Dockerfile.ubuntu_bionic | 2 +- images/promenade/Dockerfile.ubuntu_focal | 95 ++++++++ promenade/templates/include/up.sh | 9 +- requirements-direct.txt | 35 +-- requirements-frozen.txt | 211 ++++++++++-------- setup.cfg | 13 +- setup.py | 3 +- test-requirements.txt | 2 +- tests/unit/api/test_validatedesign.py | 2 +- .../unit/builder_data/simple/HostSystem.yaml | 6 +- .../builder_data/simple/armada-resources.yaml | 6 +- tools/g2/lib/config.sh | 6 +- tools/g2/lib/promenade.sh | 2 +- tools/g2/lib/registry.sh | 8 +- tools/g2/lib/virsh.sh | 6 +- tools/g2/stages/conformance.sh | 2 +- tools/g2/stages/generate-certificates.sh | 4 +- tools/g2/templates/network-config.sub | 2 +- .../bootstrap-armada-config.yaml | 6 +- tools/gate/config-templates/site-config.yaml | 2 +- tools/helm_tk.sh | 2 +- tools/install-external-deps.sh | 3 +- tools/lint_gate.sh | 6 +- tools/setup_gate.sh | 18 +- .../deploy-promenade-containerd.yaml | 31 ++- tools/zuul/playbooks/deploy-promenade.yaml | 14 +- tools/zuul/playbooks/docker-image-build.yaml | 59 ++--- .../playbooks/osh-infra-upgrade-host.yaml | 0 .../roles/deploy-apparmor/tasks/main.yaml | 41 ++++ .../roles/deploy-package/defaults/main.yml | 18 ++ .../roles/deploy-package/tasks/dist.yaml | 46 ++++ .../roles/deploy-package/tasks/pip.yaml | 27 +++ .../roles/upgrade-host/defaults/main.yml | 15 ++ .../roles/upgrade-host/tasks/main.yaml | 44 ++++ tox.ini | 41 ++-- .zuul.yaml => zuul.d/jobs.yaml | 117 ++-------- zuul.d/nodesets.yaml | 22 ++ zuul.d/project.yaml | 44 ++++ 53 files changed, 753 insertions(+), 358 deletions(-) create mode 100644 images/promenade/Dockerfile.ubuntu_focal create mode 100644 tools/zuul/playbooks/osh-infra-upgrade-host.yaml create mode 100644 tools/zuul/playbooks/roles/deploy-apparmor/tasks/main.yaml create mode 100644 tools/zuul/playbooks/roles/deploy-package/defaults/main.yml create mode 100644 tools/zuul/playbooks/roles/deploy-package/tasks/dist.yaml create mode 100644 tools/zuul/playbooks/roles/deploy-package/tasks/pip.yaml create mode 100644 tools/zuul/playbooks/roles/upgrade-host/defaults/main.yml create mode 100644 tools/zuul/playbooks/roles/upgrade-host/tasks/main.yaml rename .zuul.yaml => zuul.d/jobs.yaml (73%) create mode 100644 zuul.d/nodesets.yaml create mode 100644 zuul.d/project.yaml diff --git a/Makefile b/Makefile index 6ea4bde0..b663f2c7 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ PUSH_IMAGE ?= false # use this variable for image labels added in internal build process LABEL ?= org.airshipit.build=community COMMIT ?= $(shell git rev-parse HEAD) -DISTRO ?= ubuntu_bionic +DISTRO ?= ubuntu_focal PYTHON = python3 CHARTS := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.))) IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} @@ -43,7 +43,7 @@ docs: clean tox -e docs tests-unit: external-deps - tox -e py36 + tox -e py38 external-deps: ./tools/install-external-deps.sh @@ -51,6 +51,9 @@ external-deps: tests-pep8: tox -e pep8 +tests-freeze: + tox -e freeze + chartbanner: @echo Building charts: $(CHARTS) diff --git a/charts/promenade/Chart.yaml b/charts/promenade/Chart.yaml index 11c4a0e5..d1450068 100644 --- a/charts/promenade/Chart.yaml +++ b/charts/promenade/Chart.yaml @@ -2,3 +2,4 @@ apiVersion: v1 description: The Promenade API name: promenade version: 0.1.0 +appVersion: 1.1.0 diff --git a/doc/requirements.txt b/doc/requirements.txt index cb87a923..6e79184e 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,5 +1,5 @@ sphinx>=1.6.2 sphinx_rtd_theme==0.2.4 -falcon==1.4.1 +falcon>=1.4.1 oslo.config==7.0.0 markupsafe==2.0.1 diff --git a/doc/source/conf.py b/doc/source/conf.py index 75403236..819f4622 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -57,9 +57,9 @@ author = u'Promenade Authors' # built documents. # # The short X.Y version. -version = u'0.1.0' +version = u'0.9.0' # The full version, including alpha/beta/rc tags. -release = u'0.1.0' +release = u'0.9.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/configuration/host-system.rst b/doc/source/configuration/host-system.rst index 0118b51c..c4d3d907 100644 --- a/doc/source/configuration/host-system.rst +++ b/doc/source/configuration/host-system.rst @@ -26,7 +26,7 @@ Sample Document to run containers in Docker runtime monitoring_image: busybox:1.28.3 packages: repositories: - - deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + - deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable keys: - |- -----BEGIN PGP PUBLIC KEY BLOCK----- diff --git a/examples/basic/HostSystem.yaml b/examples/basic/HostSystem.yaml index 1b7b114a..d42ae450 100644 --- a/examples/basic/HostSystem.yaml +++ b/examples/basic/HostSystem.yaml @@ -73,6 +73,14 @@ data: mode: 0744 content: |- export KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml + - path: /etc/containerd/config.toml + mode: 0400 + content: |- + version = 2 + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"] + endpoint = ["http://registry:5000"] images: monitoring_image: &busybox busybox:1.28.3 haproxy: haproxy:1.8.3 @@ -81,8 +89,8 @@ data: packages: common: repositories: - - deb https://download.ceph.com/debian-nautilus/ bionic main - - deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + - deb https://download.ceph.com/debian-nautilus/ focal main + - deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable keys: - |- -----BEGIN PGP PUBLIC KEY BLOCK----- diff --git a/examples/basic/Kubelet.yaml b/examples/basic/Kubelet.yaml index 427544cd..4b4d6365 100644 --- a/examples/basic/Kubelet.yaml +++ b/examples/basic/Kubelet.yaml @@ -9,6 +9,8 @@ metadata: storagePolicy: cleartext data: arguments: + - --container-runtime=remote + - --container-runtime-endpoint=unix:///run/containerd/containerd.sock - --v=3 images: pause: k8s.gcr.io/pause-amd64:3.1 diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index 327cadb6..5e61c1f7 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -109,9 +109,9 @@ data: values: {} source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra + location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b dependencies: [] --- schema: armada/Chart/v1 @@ -133,9 +133,9 @@ data: values: {} source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra + location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b dependencies: [] --- schema: armada/Chart/v1 @@ -501,7 +501,7 @@ data: CLUSTER_TYPE: - k8s - bgp - IP_AUTODETECTION_METHOD: interface=ens3 + IP_AUTODETECTION_METHOD: interface=ens1 WAIT_FOR_STORAGE: "true" endpoints: @@ -533,8 +533,8 @@ data: service_calico_etcd: false source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + location: https://opendev.org/openstack/openstack-helm-infra.git + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b subpath: calico dependencies: - infra-helm-toolkit diff --git a/examples/complete/HostSystem.yaml b/examples/complete/HostSystem.yaml index 8b3741b6..adff572c 100644 --- a/examples/complete/HostSystem.yaml +++ b/examples/complete/HostSystem.yaml @@ -8,6 +8,9 @@ metadata: layer: site storagePolicy: cleartext data: + systemd_units: + kube-cgroup: + enable: true files: - path: /opt/kubernetes/bin/kubelet tar_url: https://dl.k8s.io/v1.24.4/kubernetes-node-linux-amd64.tar.gz @@ -17,6 +20,38 @@ data: tar_url: https://dl.k8s.io/v1.24.4/kubernetes-node-linux-amd64.tar.gz tar_path: kubernetes/node/bin/kubectl mode: 0555 + - path: /etc/systemd/system/kube-cgroup.service + content: | + [Unit] + Description=Create and tune cgroup for Kubernetes Pods + Requires=network-online.target + Before=kubelet.service + + [Service] + Delegate=yes + ExecStart=/usr/local/sbin/kube-cgroup.sh + + [Install] + RequiredBy=kubelet.service + mode: 0444 + - path: /usr/local/sbin/kube-cgroup.sh + mode: 0744 + content: |- + #!/bin/bash + + set -x + + KUBE_CGROUP=${KUBE_CGROUP:-"kube_whitelist"} + SYSTEMD_ABSPATH="/sys/fs/cgroup/systemd/$KUBE_CGROUP" + CPUSET_ABSPATH="/sys/fs/cgroup/cpuset/$KUBE_CGROUP" + CPU_ABSPATH="/sys/fs/cgroup/cpu/$KUBE_CGROUP" + MEM_ABSPATH="/sys/fs/cgroup/memory/$KUBE_CGROUP" + PIDS_ABSPATH="/sys/fs/cgroup/pids/$KUBE_CGROUP" + + for cg in $SYSTEMD_ABSPATH $CPUSET_ABSPATH $CPU_ABSPATH $MEM_ABSPATH $PIDS_ABSPATH + do + mkdir -p "$cg" + done - path: /etc/logrotate.d/json-logrotate mode: 0444 content: |- @@ -38,6 +73,14 @@ data: mode: 0744 content: |- export KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml + - path: /etc/containerd/config.toml + mode: 0400 + content: |- + version = 2 + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"] + endpoint = ["http://registry:5000"] images: monitoring_image: busybox:1.28.3 haproxy: haproxy:1.8.3 @@ -46,7 +89,7 @@ data: packages: common: repositories: - - deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + - deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable keys: - |- -----BEGIN PGP PUBLIC KEY BLOCK----- @@ -121,7 +164,7 @@ data: socat: socat genesis: repositories: - - deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + - deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable keys: - |- -----BEGIN PGP PUBLIC KEY BLOCK----- @@ -196,7 +239,7 @@ data: socat: socat join: repositories: - - deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + - deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable keys: - |- -----BEGIN PGP PUBLIC KEY BLOCK----- diff --git a/examples/complete/Kubelet.yaml b/examples/complete/Kubelet.yaml index 78aa09ff..66d3eefa 100644 --- a/examples/complete/Kubelet.yaml +++ b/examples/complete/Kubelet.yaml @@ -9,10 +9,9 @@ metadata: storagePolicy: cleartext data: arguments: - - --cni-bin-dir=/opt/cni/bin - - --cni-conf-dir=/etc/cni/net.d - - --network-plugin=cni - - --v=5 + - --container-runtime=remote + - --container-runtime-endpoint=unix:///run/containerd/containerd.sock + - --v=3 images: pause: k8s.gcr.io/pause-amd64:3.1 config_file_overrides: diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml index 7732c5a8..0181482d 100644 --- a/examples/complete/armada-resources.yaml +++ b/examples/complete/armada-resources.yaml @@ -150,9 +150,9 @@ data: values: {} source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra + location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b dependencies: [] --- schema: armada/Chart/v1 @@ -175,9 +175,9 @@ data: values: {} source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra + location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b dependencies: [] --- schema: armada/Chart/v1 @@ -522,7 +522,7 @@ data: CLUSTER_TYPE: - k8s - bgp - IP_AUTODETECTION_METHOD: interface=ens3 + IP_AUTODETECTION_METHOD: interface=ens1 WAIT_FOR_STORAGE: "true" endpoints: @@ -553,7 +553,7 @@ data: service_calico_etcd: false source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra + location: https://opendev.org/openstack/openstack-helm-infra.git reference: master subpath: calico dependencies: diff --git a/examples/containerd/armada-resources.yaml b/examples/containerd/armada-resources.yaml index 3b49c62e..3a42ea18 100644 --- a/examples/containerd/armada-resources.yaml +++ b/examples/containerd/armada-resources.yaml @@ -111,7 +111,7 @@ data: type: git location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b dependencies: [] --- schema: armada/Chart/v1 @@ -135,7 +135,7 @@ data: type: git location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b dependencies: [] --- schema: armada/Chart/v1 @@ -428,7 +428,7 @@ data: source: type: git location: https://opendev.org/openstack/openstack-helm-infra.git - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b subpath: calico dependencies: - infra-helm-toolkit diff --git a/examples/gate/HostSystem.yaml b/examples/gate/HostSystem.yaml index 43a466e1..9af312f8 100644 --- a/examples/gate/HostSystem.yaml +++ b/examples/gate/HostSystem.yaml @@ -73,6 +73,14 @@ data: mode: 0744 content: |- export KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml + - path: /etc/containerd/config.toml + mode: 0400 + content: |- + version = 2 + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"] + endpoint = ["http://registry:5000"] images: monitoring_image: &busybox busybox:1.28.3 haproxy: haproxy:1.8.3 @@ -81,7 +89,7 @@ data: packages: common: repositories: - - deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + - deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable keys: - |- -----BEGIN PGP PUBLIC KEY BLOCK----- diff --git a/examples/gate/Kubelet.yaml b/examples/gate/Kubelet.yaml index 7c5d95aa..4b4d6365 100644 --- a/examples/gate/Kubelet.yaml +++ b/examples/gate/Kubelet.yaml @@ -9,9 +9,8 @@ metadata: storagePolicy: cleartext data: arguments: - - --cni-bin-dir=/opt/cni/bin - - --cni-conf-dir=/etc/cni/net.d - - --network-plugin=cni + - --container-runtime=remote + - --container-runtime-endpoint=unix:///run/containerd/containerd.sock - --v=3 images: pause: k8s.gcr.io/pause-amd64:3.1 diff --git a/examples/gate/armada-resources.yaml b/examples/gate/armada-resources.yaml index 806f24e0..85944dbe 100644 --- a/examples/gate/armada-resources.yaml +++ b/examples/gate/armada-resources.yaml @@ -111,7 +111,7 @@ data: type: git location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b dependencies: [] --- schema: armada/Chart/v1 @@ -135,7 +135,7 @@ data: type: git location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b dependencies: [] --- schema: armada/Chart/v1 @@ -401,7 +401,7 @@ data: CLUSTER_TYPE: - k8s - bgp - IP_AUTODETECTION_METHOD: interface=ens3 + IP_AUTODETECTION_METHOD: interface=ens1 WAIT_FOR_STORAGE: "true" endpoints: @@ -434,7 +434,7 @@ data: source: type: git location: https://opendev.org/openstack/openstack-helm-infra.git - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b subpath: calico dependencies: - infra-helm-toolkit diff --git a/images/promenade/Dockerfile.ubuntu_bionic b/images/promenade/Dockerfile.ubuntu_bionic index a62ffaa0..485a5f17 100644 --- a/images/promenade/Dockerfile.ubuntu_bionic +++ b/images/promenade/Dockerfile.ubuntu_bionic @@ -82,7 +82,7 @@ COPY requirements-frozen.txt /opt/promenade RUN pip3 install --no-cache-dir -r requirements-frozen.txt # Setting promenade version for BPR -ENV PBR_VERSION 0.8.0 +ENV PBR_VERSION 0.9.0 COPY . /opt/promenade RUN pip3 install -e /opt/promenade diff --git a/images/promenade/Dockerfile.ubuntu_focal b/images/promenade/Dockerfile.ubuntu_focal new file mode 100644 index 00000000..3f8d0727 --- /dev/null +++ b/images/promenade/Dockerfile.ubuntu_focal @@ -0,0 +1,95 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG FROM=ubuntu:20.04 +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://airship-promenade.readthedocs.org' \ + org.opencontainers.image.source='https://opendev.org/airship/promenade' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +VOLUME /etc/promenade +VOLUME /target + +RUN mkdir /opt/promenade +WORKDIR /opt/promenade + +ENV PORT 9000 +EXPOSE $PORT + +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC + +ENTRYPOINT ["/opt/promenade/entrypoint.sh"] + +RUN set -ex \ + && apt-get update -qq \ + && apt-get install --no-install-recommends -y \ + automake \ + ca-certificates \ + curl \ + dnsutils \ + gcc \ + git \ + gpg \ + gpg-agent \ + libpcre3-dev \ + libtool \ + make \ + python3-dev \ + python3-pip \ + python3-setuptools \ + rsync \ + && ln -s /usr/bin/python3 /usr/bin/python \ + && curl -Lo /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \ + && chmod 555 /usr/local/bin/cfssl \ + && python3 -m pip install -U pip \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && useradd -u 1000 -g users -d /opt/promenade promenade \ + && rm -rf /var/lib/apt/lists/* + +# Install LibYAML +ENV LD_LIBRARY_PATH=/usr/local/lib + +ARG LIBYAML_VERSION=0.2.5 +RUN set -ex \ + && git clone https://github.com/yaml/libyaml.git \ + && cd libyaml \ + && git checkout $LIBYAML_VERSION \ + && ./bootstrap \ + && ./configure \ + && make \ + && make install \ + && cd .. \ + && rm -fr libyaml + +COPY requirements-frozen.txt /opt/promenade +RUN pip3 install --no-cache-dir -r requirements-frozen.txt + +# Setting promenade version for BPR +ENV PBR_VERSION 0.9.0 + +COPY . /opt/promenade +RUN pip3 install --verbose --editable /opt/promenade \ + && echo "/opt/promenade" \ + > /usr/local/lib/python3.8/dist-packages/promenade.pth + +USER promenade diff --git a/promenade/templates/include/up.sh b/promenade/templates/include/up.sh index e057ce8c..785f7544 100644 --- a/promenade/templates/include/up.sh +++ b/promenade/templates/include/up.sh @@ -108,7 +108,11 @@ while true; do done while true; do - if ! DEBIAN_FRONTEND=noninteractive apt-get install -o Dpkg::Options::="--force-confold" -y --no-install-recommends \ + if ! DEBIAN_FRONTEND=noninteractive apt-get install \ + -o Dpkg::Options::="--force-confold" \ + -y \ + --allow-downgrades \ + --no-install-recommends \ {%- for role in roles %} {%- for package in config.get_path('HostSystem:packages.' + role + '.required',{}).values() %} {{ package }} \ @@ -129,6 +133,9 @@ while true; do fi done +sed -i -e 's/slot //g' /lib/systemd/network/99-default.link +DEBIAN_FRONTEND=noninteractive apt-get upgrade -o Dpkg::Options::="--force-confold" -q -y --allow-downgrades --no-install-recommends + # Start core processes # set +x diff --git a/requirements-direct.txt b/requirements-direct.txt index 231dd4de..4155bc32 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -1,18 +1,19 @@ -beaker==1.10.0 -click==6.7 -falcon==1.4.1 -jinja2==2.10 -jsonpath-ng==1.4.3 +Beaker==1.12.0 +click==8.1.3 +falcon==3.1.1 +Jinja2==3.1.2 +jsonpath-ng==1.5.3 jsonschema==3.2.0 -keystoneauth1==3.18.0 -keystonemiddleware==5.3.0 -setuptools==40.4.3 -kubernetes~=24.2.0 -oslo.context==2.21.0 -oslo.policy==1.40.1 -pastedeploy==1.5.2 -pylibyaml~=0.1 -pyyaml~=5.1 -requests==2.22.0 -uwsgi==2.0.19.1 -git+https://opendev.org/airship/deckhand.git@5cd799cc5d04527ac782270008ff647b3779ff05#egg=deckhand +keystoneauth1==5.1.1 +keystonemiddleware==10.2.0 +setuptools==67.0.0 +kubernetes==24.2.0 +oslo.context==5.0.0 +oslo.policy==4.0.0 +PasteDeploy==3.0.1 +pylibyaml==0.1.0 +PyYAML==5.4.1 +requests==2.28.2 +uWSGI==2.0.21 +Deckhand @ git+https://opendev.org/airship/deckhand.git@70aa35a396d5f76753616f5289228f9c2b0e7ec7 +# Deckhand @ git+https://review.opendev.org/airship/deckhand@refs/changes/93/869293/222#egg=deckhand \ No newline at end of file diff --git a/requirements-frozen.txt b/requirements-frozen.txt index a73c6302..85eac127 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,115 +1,136 @@ -alembic==1.0.1 -amqp==2.6.0 -asn1crypto==1.3.0 -Babel==2.8.0 -Beaker==1.10.0 -cachetools==4.1.0 -certifi==2020.4.5.2 -cffi==1.14.0 -chardet==3.0.4 -click==6.7 -cliff==3.1.0 -cmd2==0.8.9 -cryptography==3.1.1 -debtcollector==1.22.0 -pbr==5.4.5 -Deckhand @ git+https://opendev.org/airship/deckhand.git@5cd799cc5d04527ac782270008ff647b3779ff05#egg=deckhand -decorator==4.4.2 -deepdiff==3.3.0 -dnspython==1.16.0 -dogpile.cache==0.9.2 -eventlet==0.25.2 +alabaster==0.7.13 +alembic==1.7.1 +amqp==5.0.8 +attrs==22.2.0 +autopage==0.5.1 +Babel==2.11.0 +bcrypt==4.0.1 +Beaker==1.12.0 +cachetools==5.3.0 +certifi==2022.12.7 +cffi==1.15.1 +charset-normalizer==3.0.1 +click==8.1.3 +cliff==4.2.0 +cmd2==2.4.3 +cryptography==3.4.8 +debtcollector==2.5.0 +Deckhand @ git+https://opendev.org/airship/deckhand.git@70aa35a396d5f76753616f5289228f9c2b0e7ec7 +decorator==5.1.1 +deepdiff==5.8.1 +dnspython==2.3.0 +docutils==0.17.1 +dogpile.cache==1.1.8 +entrypoints==0.3 +eventlet==0.33.3 extras==1.0.0 -falcon==1.4.1 -fasteners==0.15 +falcon==3.1.1 +fasteners==0.18 fixtures==3.0.0 -flake8===3.7.9 -futurist==1.10.0 -google-auth==1.16.1 -greenlet==0.4.16 +flake8==3.7.9 +future==0.18.3 +futurist==2.4.1 +google-auth==2.16.1 +greenlet==2.0.2 hacking==3.0.1 -idna==2.8 -importlib-metadata==1.6.1 -ipaddress==1.0.23 -iso8601==0.1.12 -Jinja2==2.10 -jsonpath-ng==1.4.3 -jsonpickle==1.4.1 +idna==3.4 +imagesize==1.4.1 +importlib-metadata==6.0.0 +importlib-resources==5.12.0 +iso8601==1.1.0 +Jinja2==3.1.2 +jsonpath-ng==1.5.3 +jsonpickle==3.0.1 jsonschema==3.2.0 -keystoneauth1==3.18.0 -keystonemiddleware==5.3.0 -kombu==4.6.10 -kubernetes==24.2.0; python_version >= '3.6' -linecache2==1.0.0 -Mako==1.1.3 -MarkupSafe==1.1.1 +keystoneauth1==5.1.1 +keystonemiddleware==10.2.0 +kombu==5.1.0 +kubernetes==24.2.0 +Mako==1.2.4 +MarkupSafe==2.1.2 mccabe==0.6.1 -monotonic==1.5 -msgpack==1.0.0 -netaddr==0.7.19 -netifaces==0.10.9 -networkx==2.2 +msgpack==1.0.4 +netaddr==0.8.0 +netifaces==0.11.0 +networkx==2.6.2 +oauthlib==3.2.2 +ordered-set==4.1.0 os-service-types==1.7.0 -oslo.cache==1.38.1 -oslo.concurrency==3.28.1 -oslo.config==7.0.0 -oslo.context==2.21.0 -oslo.db==4.41.1 -oslo.i18n==3.25.1 -oslo.log==3.45.2 -oslo.messaging==9.1.1 -oslo.middleware==3.36.0 -oslo.policy==1.40.1 -oslo.serialization==2.29.2 -oslo.service==1.41.1 -oslo.utils==3.42.1 -Paste==3.0.1 -PasteDeploy==1.5.2 +oslo.cache==2.8.2 +oslo.concurrency==4.4.1 +oslo.config==8.7.1 +oslo.context==5.0.0 +oslo.db==11.0.0 +oslo.i18n==6.0.0 +oslo.log==4.6.0 +oslo.messaging==12.9.4 +oslo.metrics==0.6.0 +oslo.middleware==4.4.0 +oslo.policy==4.0.0 +oslo.serialization==4.2.0 +oslo.service==3.1.1 +oslo.utils==4.10.2 +packaging==23.0 +Paste==3.5.0 +PasteDeploy==3.0.1 +pbr==5.6.0 ply==3.11 -prettytable==0.7.2 -psycopg2==2.7.5 +prettytable==3.6.0 +prometheus-client==0.16.0 +psycopg2-binary==2.9.5 pyasn1==0.4.8 pyasn1-modules==0.2.8 -pycadf==2.10.0 +pycadf==3.1.1 pycodestyle==2.5.0 -pycparser==2.20 -pyflakes==2.1.0 +pycparser==2.21 +pyflakes==2.1.1 +Pygments==2.14.0 pyinotify==0.9.6 pylibyaml==0.1.0 -pyparsing==2.4.7 -pyperclip==1.8.0 -python-barbicanclient==4.7.0 -python-dateutil==2.8.1 -python-editor==1.0.4 +pyparsing==3.0.9 +pyperclip==1.8.2 +pyrsistent==0.19.3 +python-barbicanclient==5.2.0 +python-dateutil==2.8.2 python-keystoneclient==3.22.0 python-memcached==1.59 -python-mimeparse==1.6.0 -pytz==2020.1 +python-subunit==1.4.2 +pytz==2022.7.1 PyYAML==5.4.1 repoze.lru==0.7 -requests==2.22.0 -rfc3986==1.4.0 -Routes==2.4.1 -rsa==4.0 -six==1.15.0 -SQLAlchemy==1.3.17 +requests==2.28.2 +requests-oauthlib==1.3.1 +rfc3986==2.0.0 +Routes==2.5.1 +rsa==4.9 +six==1.16.0 +snowballstemmer==2.2.0 +Sphinx==5.3.0 +sphinx-rtd-theme==1.1.1 +sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +SQLAlchemy==1.4.23 sqlalchemy-migrate==0.13.0 -sqlparse==0.3.1 -statsd==3.3.0 -stevedore==1.32.0 +sqlparse==0.4.3 +statsd==4.0.1 +stestr==3.2.0 +stevedore==4.1.1 Tempita==0.5.2 testresources==2.0.1 testscenarios==0.5.0 -testtools==2.4.0 -traceback2==1.4.0 -unittest2==1.1.0 -urllib3==1.25.9 -uWSGI==2.0.19.1 -vine==1.3.0 -wcwidth==0.2.4 -WebOb==1.8.6 -websocket-client==0.57.0 -Werkzeug==0.16.1 -wrapt==1.12.1 -yappi==1.2.5 -zipp==1.2.0 +testtools==2.5.0 +urllib3==1.26.6 +uWSGI==2.0.21 +vine==5.0.0 +voluptuous==0.13.1 +wcwidth==0.2.6 +WebOb==1.8.7 +websocket-client==1.5.1 +Werkzeug==2.0.1 +wrapt==1.14.1 +yappi==1.4.0 +zipp==3.14.0 diff --git a/setup.cfg b/setup.cfg index f83240c3..8fb240f3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,12 +1,10 @@ [metadata] name = promenade -summary = 'Promenade is a tool for bootstrapping a resilient ' - 'Kubernetes cluster and managing its life-cycle via ' - 'Helm charts. -description-file = README.md +summary = Promenade is a tool for bootstrapping a resilient kubernetes cluster and managing its life-cycle via helm charts. +description_file = README.md author = The Airship Authors -author-email = airship-discuss@lists.airshipit.org -home-page = https://opendev.org/airship/promenade +author_email = airship-discuss@lists.airshipit.org +home_page = https://opendev.org/airship/promenade classifier = Intended Audience :: Information Technology @@ -14,8 +12,7 @@ classifier = License :: OSI Approved :: Apache Software License Operating System :: POSIX :: Linux Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.8 [files] packages = diff --git a/setup.py b/setup.py index d306afbe..d31b9fa7 100644 --- a/setup.py +++ b/setup.py @@ -18,5 +18,6 @@ from setuptools import setup setup( setup_requires=['setuptools>=17.1', 'pbr>=2.0.0'], - pbr=True + pbr=True, + packages=['promenade'] ) diff --git a/test-requirements.txt b/test-requirements.txt index 09185210..d47b85c5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ pytest -flake8===3.7.9 +flake8==3.7.9 bandit>=1.5 yapf==0.24.0 diff --git a/tests/unit/api/test_validatedesign.py b/tests/unit/api/test_validatedesign.py index 4443e5ba..250bb788 100644 --- a/tests/unit/api/test_validatedesign.py +++ b/tests/unit/api/test_validatedesign.py @@ -168,7 +168,7 @@ VALID_DOCS = [ '-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth\nlqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh\n38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq\nL4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7\nUrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N\ncmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht\nku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo\nvZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD\nG7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ\nXYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj\nq5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB\ntCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3\nBBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO\nv82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd\ntI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk\njonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m\n6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P\nXRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc\nFYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8\ng383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm\nebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh\n9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5\nG34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW\nFPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB\nEADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF\nM2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx\nQ+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu\nw6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk\nz2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8\neRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb\nVfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa\n1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X\nzJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ\npqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7\nZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ\nBQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY\n1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp\nYcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI\nmD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES\nKA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7\nJIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ\ncM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0\n6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5\nU888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z\nVfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f\nirP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk\nSVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz\nQhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W\n9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw\n24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe\ndKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y\nVm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR\nH3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh\n/nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ\nM8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S\nxx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O\njCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG\nYT90qFF93M3v01BbxP+EIY2/9tiIPbrd\n=0YYh\n-----END PGP PUBLIC KEY BLOCK-----' ], 'repositories': [ - 'deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable' + 'deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable' ], 'required': { 'docker': 'docker-ce=5:19.03.8~3-0~ubuntu-bionic', diff --git a/tests/unit/builder_data/simple/HostSystem.yaml b/tests/unit/builder_data/simple/HostSystem.yaml index 85234dd0..084e0714 100644 --- a/tests/unit/builder_data/simple/HostSystem.yaml +++ b/tests/unit/builder_data/simple/HostSystem.yaml @@ -42,7 +42,7 @@ data: packages: common: repositories: - - deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + - deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable keys: - |- -----BEGIN PGP PUBLIC KEY BLOCK----- @@ -116,7 +116,7 @@ data: socat: socat=1.7.3.1-1 genesis: repositories: - - deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + - deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable keys: - |- -----BEGIN PGP PUBLIC KEY BLOCK----- @@ -190,7 +190,7 @@ data: socat: socat=1.7.3.1-1 join: repositories: - - deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + - deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable keys: - |- -----BEGIN PGP PUBLIC KEY BLOCK----- diff --git a/tests/unit/builder_data/simple/armada-resources.yaml b/tests/unit/builder_data/simple/armada-resources.yaml index 29e2f800..fa1a472a 100644 --- a/tests/unit/builder_data/simple/armada-resources.yaml +++ b/tests/unit/builder_data/simple/armada-resources.yaml @@ -110,7 +110,7 @@ data: values: {} source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra + location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit reference: master dependencies: [] @@ -135,7 +135,7 @@ data: values: {} source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra + location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit reference: master dependencies: [] @@ -439,7 +439,7 @@ data: service_calico_etcd: false source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra + location: https://opendev.org/openstack/openstack-helm-infra.git reference: master subpath: calico dependencies: diff --git a/tools/g2/lib/config.sh b/tools/g2/lib/config.sh index e17aef64..1a6229f6 100644 --- a/tools/g2/lib/config.sh +++ b/tools/g2/lib/config.sh @@ -1,8 +1,8 @@ export TEMP_DIR=${TEMP_DIR:-$(mktemp -d)} -export BASE_IMAGE_SIZE=${BASE_IMAGE_SIZE:-344784896} -export BASE_IMAGE_URL=${BASE_IMAGE_URL:-https://cloud-images.ubuntu.com/releases/bionic/release/ubuntu-18.04-server-cloudimg-amd64.img} +export BASE_IMAGE_SIZE=${BASE_IMAGE_SIZE:-644612096} +export BASE_IMAGE_URL=${BASE_IMAGE_URL:-https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64.img} export IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/airshipit/promenade:master} -export IMAGE_PROMENADE_DISTRO=${IMAGE_PROMENADE_DISTRO:-ubuntu_bionic} +export IMAGE_PROMENADE_DISTRO=${IMAGE_PROMENADE_DISTRO:-ubuntu_focal} export NGINX_DIR="${TEMP_DIR}/nginx" export NGINX_URL="http://192.168.77.1:7777" export PROMENADE_BASE_URL="http://promenade-api.ucp.svc.cluster.local" diff --git a/tools/g2/lib/promenade.sh b/tools/g2/lib/promenade.sh index 13eec04e..13b2b8db 100644 --- a/tools/g2/lib/promenade.sh +++ b/tools/g2/lib/promenade.sh @@ -11,7 +11,7 @@ promenade_render_curl_url() { USE_DECKHAND=${2} DECKHAND_REVISION=${3} shift 3 - LABELS=(${@}) + LABELS=("${@}") LABEL_PARAMS= for label in "${LABELS[@]}"; do diff --git a/tools/g2/lib/registry.sh b/tools/g2/lib/registry.sh index bc5e5eb7..c63aff60 100644 --- a/tools/g2/lib/registry.sh +++ b/tools/g2/lib/registry.sh @@ -1,13 +1,13 @@ registry_down() { REGISTRY_ID=$(docker ps -qa -f name=registry) - if [[ ! -z ${REGISTRY_ID} ]]; then + if [[ -n ${REGISTRY_ID} ]]; then log Removing docker registry docker rm -fv "${REGISTRY_ID}" &>> "${LOG_FILE}" fi } registry_list_images() { - FILES=($(config_configuration | xargs -n 1 -I DIRNAME find DIRNAME -type f -name '*.yaml' | grep -v PKICatalog)) + FILES=("$(config_configuration | xargs -I dirname find dirname -type f -name '*.yaml' | grep -v PKICatalog)") HOSTNAME_REGEX='[a-zA-Z0-9][a-zA-Z0-9_-]{0,62}' DOMAIN_NAME_REGEX="${HOSTNAME_REGEX}(\.${HOSTNAME_REGEX})*" @@ -45,7 +45,7 @@ registry_populate() { } registry_replace_references() { - FILES=(${@}) + FILES=("${@}") for image in $(registry_list_images); do sed -i "s;${image}\$;registry:5000/${image};g" "${FILES[@]}" done @@ -55,7 +55,7 @@ registry_up() { log Validating local registry is up REGISTRY_ID=$(docker ps -qa -f name=registry) RUNNING_REGISTRY_ID=$(docker ps -q -f name=registry) - if [[ -z ${RUNNING_REGISTRY_ID} && ! -z ${REGISTRY_ID} ]]; then + if [[ -z ${RUNNING_REGISTRY_ID} && -n ${REGISTRY_ID} ]]; then log Removing stopped docker registry docker rm -fv "${REGISTRY_ID}" &>> "${LOG_FILE}" fi diff --git a/tools/g2/lib/virsh.sh b/tools/g2/lib/virsh.sh index d1bcd21d..285773eb 100644 --- a/tools/g2/lib/virsh.sh +++ b/tools/g2/lib/virsh.sh @@ -22,6 +22,7 @@ img_base_declare() { } iso_gen() { + set -x NAME=${1} if virsh vol-key --pool "${VIRSH_POOL}" --vol "cloud-init-${NAME}.iso" &> /dev/null; then @@ -67,6 +68,7 @@ iso_gen() { --vol "cloud-init-${NAME}.iso" \ --file "${ISO_DIR}/cidata.iso" } &>> "${LOG_FILE}" + set +x } iso_path() { @@ -125,11 +127,12 @@ vm_create() { DISK_OPTS="bus=virtio,cache=directsync,discard=unmap,format=qcow2" virt-install \ --name "${NAME}" \ + --osinfo ubuntu20.04 \ --virt-type kvm \ --cpu host,+x2apic,-avx2 \ --graphics vnc,listen=0.0.0.0 \ --noautoconsole \ - --network "network=promenade,model=virtio" \ + --network "network=promenade,model=e1000" \ --vcpus "$(config_vm_vcpus)" \ --memory "$(config_vm_memory)" \ --import \ @@ -140,6 +143,7 @@ vm_create() { ssh_cmd "${NAME}" sync # docker enables forwarding, containerd - does not ssh_cmd "${NAME}" sysctl net.ipv4.conf.all.forwarding=1 + ssh_cmd "${NAME}" sysctl net.ipv4.conf.all.rp_filter=1 } vm_create_all() { diff --git a/tools/g2/stages/conformance.sh b/tools/g2/stages/conformance.sh index 986000d4..b14a761f 100755 --- a/tools/g2/stages/conformance.sh +++ b/tools/g2/stages/conformance.sh @@ -20,7 +20,7 @@ else fi FILENAME=$(ssh_cmd "${GENESIS_NAME}" ls /mnt/sonobuoy || echo "") -if [[ ! -z ${FILENAME} ]]; then +if [[ -n ${FILENAME} ]]; then if rsync_cmd "${GENESIS_NAME}:/mnt/sonobuoy/${FILENAME}" "${WORKSPACE}/conformance/sonobuoy.tgz"; then tar xf "${WORKSPACE}/conformance/sonobuoy.tgz" -C "${WORKSPACE}/conformance" fi diff --git a/tools/g2/stages/generate-certificates.sh b/tools/g2/stages/generate-certificates.sh index 920c9eee..65ff6e9d 100755 --- a/tools/g2/stages/generate-certificates.sh +++ b/tools/g2/stages/generate-certificates.sh @@ -50,7 +50,7 @@ rm -f "${OUTPUT_FILE}" for source_dir in $(config_configuration); do log Copying configuration from "${source_dir}" - for filename in ${WORKSPACE}/${source_dir}/*.yaml; do + for filename in "${WORKSPACE}"/"${source_dir}"/*.yaml; do if should_include_filename "${filename}"; then log Including config from "$filename" cat "${filename}" >> "${OUTPUT_FILE}" @@ -68,7 +68,7 @@ log "Setting up local caches.." nginx_cache_and_replace_tar_urls "${OUTPUT_DIR}"/*.yaml registry_replace_references "${OUTPUT_DIR}"/*.yaml -FILES=($(ls "${OUTPUT_DIR}")) +FILES=("$(ls "${OUTPUT_DIR}")") log Generating certificates docker run --rm -t \ diff --git a/tools/g2/templates/network-config.sub b/tools/g2/templates/network-config.sub index 8df1b7e2..7562ca5d 100644 --- a/tools/g2/templates/network-config.sub +++ b/tools/g2/templates/network-config.sub @@ -2,7 +2,7 @@ version: 1 config: - type: physical - name: ens3 + name: ens1 subnets: - type: static address: ${BR_IP_NODE}/24 diff --git a/tools/gate/config-templates/bootstrap-armada-config.yaml b/tools/gate/config-templates/bootstrap-armada-config.yaml index f237875f..431870cd 100644 --- a/tools/gate/config-templates/bootstrap-armada-config.yaml +++ b/tools/gate/config-templates/bootstrap-armada-config.yaml @@ -105,9 +105,9 @@ data: values: {} source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra + location: https://opendev.org/openstack/openstack-helm-infra.git subpath: helm-toolkit - reference: ee331fd0d38e3584fecbefbf6218ba0c8e112338 + reference: fa8916f5bcc8cbf064a387569e2630b7bbf0b49b dependencies: [] --- schema: armada/Chart/v1 @@ -479,7 +479,7 @@ data: service_calico_etcd: false source: type: git - location: https://git.openstack.org/openstack/openstack-helm-infra + location: https://opendev.org/openstack/openstack-helm-infra.git reference: master subpath: calico dependencies: diff --git a/tools/gate/config-templates/site-config.yaml b/tools/gate/config-templates/site-config.yaml index 015e3d19..09e37ff9 100644 --- a/tools/gate/config-templates/site-config.yaml +++ b/tools/gate/config-templates/site-config.yaml @@ -75,7 +75,7 @@ data: helm: ${IMAGE_HELM} packages: repositories: - - deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable + - deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable keys: - |- -----BEGIN PGP PUBLIC KEY BLOCK----- diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index 50416708..a8d90ec1 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -17,7 +17,7 @@ set -eux HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"} -HTK_STABLE_COMMIT=${HTK_COMMIT:-"5c4056ad341afcc577e63902b6ddbfb222d757e1"} +HTK_STABLE_COMMIT=${HTK_COMMIT:-"fa8916f5bcc8cbf064a387569e2630b7bbf0b49b"} diff --git a/tools/install-external-deps.sh b/tools/install-external-deps.sh index 053ad953..8a3aa24d 100755 --- a/tools/install-external-deps.sh +++ b/tools/install-external-deps.sh @@ -5,7 +5,7 @@ set -ex CFSSL_URL=${CFSSL_URL:-https://pkg.cfssl.org/R1.2/cfssl_linux-amd64} -if [[ ! $(which cfssl) ]]; then +if [[ ! $(command -v cfssl) ]]; then TMP_DIR=$(mktemp -d) pushd "${TMP_DIR}" curl -Lo cfssl "${CFSSL_URL}" @@ -13,4 +13,5 @@ if [[ ! $(which cfssl) ]]; then sudo mv cfssl /usr/local/bin/ popd rm -rf "${TMP_DIR}" + cfssl version fi diff --git a/tools/lint_gate.sh b/tools/lint_gate.sh index f86cceec..6ee225ca 100755 --- a/tools/lint_gate.sh +++ b/tools/lint_gate.sh @@ -7,12 +7,12 @@ WORKSPACE=$(realpath "${SCRIPT_DIR}/..") for manifest in $(find "${WORKSPACE}/tools/g2/manifests" -type f | sort); do echo Checking "${manifest}" - python -m jsonschema "${WORKSPACE}/tools/g2/manifest-schema.json" -i "${manifest}" + python3 -m jsonschema "${WORKSPACE}/tools/g2/manifest-schema.json" -i "${manifest}" done -if [[ -x $(which shellcheck) ]]; then +if [[ -x "$(command -v shellcheck)" ]]; then echo Checking shell scripts.. - shellcheck -s bash -e SC1090 -e SC2162 -e SC2164 -e SC2128 -e SC2029 "${WORKSPACE}"/tools/cleanup.sh "${WORKSPACE}"/tools/*gate*.sh "${WORKSPACE}"/tools/g2/stages/* "${WORKSPACE}"/tools/g2/lib/* "${WORKSPACE}"/tools/install-external-deps.sh + shellcheck -s bash -e SC1091 -e SC1090 -e SC2162 -e SC2164 -e SC2128 -e SC2029 "${WORKSPACE}"/tools/cleanup.sh "${WORKSPACE}"/tools/*gate*.sh "${WORKSPACE}"/tools/g2/stages/* "${WORKSPACE}"/tools/g2/lib/* "${WORKSPACE}"/tools/install-external-deps.sh else echo No shellcheck executable found. Please, install it. exit 1 diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index 35ae3679..c74c5d8b 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -19,7 +19,7 @@ REQUIRE_RELOG=0 log_stage_header "Installing Packages" export DEBIAN_FRONTEND=noninteractive sudo apt-get update -qq -sudo apt-get install -q -y --no-install-recommends \ +sudo apt-get install -q -y --no-install-recommends --allow-downgrades \ apt-transport-https \ build-essential \ ca-certificates \ @@ -28,12 +28,15 @@ sudo apt-get install -q -y --no-install-recommends \ genisoimage \ jq \ libstring-shellquote-perl \ - libvirt-bin \ python3-dev \ + software-properties-common \ qemu-kvm \ qemu-utils \ - software-properties-common \ - virtinst + virt-manager \ + libvirt-daemon-system \ + virtinst \ + libvirt-clients \ + bridge-utils # Install the docker gpg key & Add the repository curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - @@ -44,10 +47,13 @@ sudo add-apt-repository \ sudo apt-get update -qq # Remove old versions of docker, if installed -sudo apt-get remove -q -y docker docker-engine docker.io +sudo apt-get remove -q -y docker docker-engine docker.io --allow-downgrades \ # Install docker sudo apt-get install -q -y --no-install-recommends \ - docker-ce + docker-ce \ + docker-ce-cli \ + docker-buildx-plugin \ + containerd.io # Set up proxy when using docker_image in yamls sudo mkdir -p /etc/systemd/system/docker.service.d/ diff --git a/tools/zuul/playbooks/deploy-promenade-containerd.yaml b/tools/zuul/playbooks/deploy-promenade-containerd.yaml index 89d816cf..27a8414b 100644 --- a/tools/zuul/playbooks/deploy-promenade-containerd.yaml +++ b/tools/zuul/playbooks/deploy-promenade-containerd.yaml @@ -19,9 +19,33 @@ PROMENADE_TMP_LOCAL: "cache" DISTRO: "{{ distro }}" become: true + tasks: - - name: Install docker - command: apt-get install docker.io resolvconf -y + + - name: Generate configuration files + shell: | + set -xe; + sysctl net.ipv4.conf.all.forwarding=1 + sysctl net.ipv4.conf.all.rp_filter=1 + args: + chdir: /root + executable: /bin/bash + environment: "{{env}}" + + - name: Ensure pip + include_role: + name: ensure-pip + + - name: Clear firewall + include_role: + name: clear-firewall + + - name: Ensure docker + include_role: + name: ensure-docker + + - name: Install resolvconf + command: apt-get install resolvconf -y - name: Setup Apparmor shell: | @@ -31,6 +55,9 @@ chdir: "{{ zuul.projects['opendev.org/openstack/openstack-helm-infra'].src_dir }}" executable: /bin/bash + - name: List interfaces + command: ip addr show + - name: Generate configuration files shell: | set -xe; diff --git a/tools/zuul/playbooks/deploy-promenade.yaml b/tools/zuul/playbooks/deploy-promenade.yaml index 2d1be239..2e831f81 100644 --- a/tools/zuul/playbooks/deploy-promenade.yaml +++ b/tools/zuul/playbooks/deploy-promenade.yaml @@ -18,9 +18,21 @@ NO_PROXY: "" DISTRO: "{{ distro }}" become: true + + roles: + - deploy-apparmor + - ensure-docker + tasks: - name: Install docker - command: apt-get install docker.io resolvconf -y + command: | + apt-get install \ + docker-ce \ + docker-ce-cli \ + docker-buildx-plugin \ + containerd.io \ + resolvconf \ + -y - name: Setup Apparmor shell: | diff --git a/tools/zuul/playbooks/docker-image-build.yaml b/tools/zuul/playbooks/docker-image-build.yaml index 0858038d..4a8786b6 100644 --- a/tools/zuul/playbooks/docker-image-build.yaml +++ b/tools/zuul/playbooks/docker-image-build.yaml @@ -1,52 +1,19 @@ - hosts: all + tasks: - include_vars: vars.yaml - - name: Install Docker (Debian) - when: ansible_os_family == 'Debian' - block: - - file: - path: "{{ item }}" - state: directory - with_items: - - /etc/docker/ - - /etc/systemd/system/docker.service.d/ - - /var/lib/docker/ - - mount: - path: /var/lib/docker/ - src: tmpfs - fstype: tmpfs - opts: size=25g - state: mounted - - copy: "{{ item }}" - with_items: - - content: "{{ docker_daemon | to_json }}" - dest: /etc/docker/daemon.json - - src: files/docker-systemd.conf - dest: /etc/systemd/system/docker.service.d/ - - apt_key: - url: https://download.docker.com/linux/ubuntu/gpg - - apt_repository: - repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker/{{ ansible_distribution_release }} {{ ansible_distribution_release }} stable - - apt: - name: "{{ item }}" - allow_unauthenticated: True - with_items: - - docker-ce - - python3-pip - - python3-setuptools - - pip: - name: docker - version: 2.7.0 - executable: pip3 - # NOTE(SamYaple): Allow all connections from containers to host so the - # containers can access the http server for git and wheels - - iptables: - action: insert - chain: INPUT - in_interface: docker0 - jump: ACCEPT - become: True + - name: Ensure pip + include_role: + name: ensure-pip + + - name: Clear firewall + include_role: + name: clear-firewall + + - name: Ensure docker + include_role: + name: ensure-docker - name: Debug tag generation inputs block: @@ -62,7 +29,7 @@ msg: "{{ tags | to_json }}" - name: Determine tags - shell: echo '{{ tags | to_json }}' | python {{ zuul.project.src_dir }}/tools/image_tags.py + shell: echo '{{ tags | to_json }}' | python3 {{ zuul.project.src_dir }}/tools/image_tags.py environment: BRANCH: "{{ zuul.branch | default('') }}" CHANGE: "{{ zuul.change | default('') }}" diff --git a/tools/zuul/playbooks/osh-infra-upgrade-host.yaml b/tools/zuul/playbooks/osh-infra-upgrade-host.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tools/zuul/playbooks/roles/deploy-apparmor/tasks/main.yaml b/tools/zuul/playbooks/roles/deploy-apparmor/tasks/main.yaml new file mode 100644 index 00000000..16e13cd9 --- /dev/null +++ b/tools/zuul/playbooks/roles/deploy-apparmor/tasks/main.yaml @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- block: + - name: ensuring AppArmor is deployed on host + when: ansible_distribution == 'Ubuntu' + include_role: + name: deploy-package + tasks_from: dist + vars: + packages: + deb: + - apparmor + - apt-transport-https + - ca-certificates + - curl + - software-properties-common + + - name: "Enable AppArmor" + when: ansible_distribution == 'Ubuntu' + become: true + become_user: root + shell: |- + set -xe + systemctl enable apparmor + systemctl start apparmor + systemctl status apparmor.service + args: + executable: /bin/bash + ignore_errors: True +... diff --git a/tools/zuul/playbooks/roles/deploy-package/defaults/main.yml b/tools/zuul/playbooks/roles/deploy-package/defaults/main.yml new file mode 100644 index 00000000..b1a6fabd --- /dev/null +++ b/tools/zuul/playbooks/roles/deploy-package/defaults/main.yml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +proxy: + http: null + https: null + noproxy: null +... diff --git a/tools/zuul/playbooks/roles/deploy-package/tasks/dist.yaml b/tools/zuul/playbooks/roles/deploy-package/tasks/dist.yaml new file mode 100644 index 00000000..73939ffd --- /dev/null +++ b/tools/zuul/playbooks/roles/deploy-package/tasks/dist.yaml @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: managing distro packages for ubuntu + become: true + become_user: root + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + vars: + state: present + apt: + name: "{{ item }}" + state: "{{ state }}" + with_items: "{{ packages.deb }}" + +- name: managing distro packages for centos + become: true + become_user: root + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + vars: + state: present + yum: + name: "{{ item }}" + state: "{{ state }}" + with_items: "{{ packages.rpm }}" + +- name: managing distro packages for fedora + become: true + become_user: root + when: ansible_distribution == 'Fedora' + vars: + state: present + dnf: + name: "{{ item }}" + state: "{{ state }}" + with_items: "{{ packages.rpm }}" +... diff --git a/tools/zuul/playbooks/roles/deploy-package/tasks/pip.yaml b/tools/zuul/playbooks/roles/deploy-package/tasks/pip.yaml new file mode 100644 index 00000000..0b2a4836 --- /dev/null +++ b/tools/zuul/playbooks/roles/deploy-package/tasks/pip.yaml @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: managing pip packages + become: true + become_user: root + environment: + http_proxy: "{{ proxy.http }}" + https_proxy: "{{ proxy.https }}" + no_proxy: "{{ proxy.noproxy }}" + vars: + state: present + pip: + name: "{{ item }}" + state: "{{ state }}" + with_items: "{{ packages }}" +... diff --git a/tools/zuul/playbooks/roles/upgrade-host/defaults/main.yml b/tools/zuul/playbooks/roles/upgrade-host/defaults/main.yml new file mode 100644 index 00000000..93b068cd --- /dev/null +++ b/tools/zuul/playbooks/roles/upgrade-host/defaults/main.yml @@ -0,0 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +ubuntu_kernel_hwe: false +... diff --git a/tools/zuul/playbooks/roles/upgrade-host/tasks/main.yaml b/tools/zuul/playbooks/roles/upgrade-host/tasks/main.yaml new file mode 100644 index 00000000..dacf92f7 --- /dev/null +++ b/tools/zuul/playbooks/roles/upgrade-host/tasks/main.yaml @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Upgrade to HWE kernel on Ubuntu Hosts + when: + - ansible_distribution == 'Ubuntu' + - ubuntu_kernel_hwe == true + block: + - name: Deploy HWE kernel on Ubuntu Hosts + include_role: + name: deploy-package + tasks_from: dist + vars: + packages: + deb: + - linux-generic-hwe-20.04 + - name: Reboot Host following kernel upgrade + shell: sleep 2 && reboot + become: yes + async: 30 + poll: 0 + ignore_errors: true + args: + executable: /bin/bash + - name: Wait for hosts to come up following reboot + wait_for: + host: '{{ hostvars[item].ansible_host }}' + port: 22 + state: started + delay: 60 + timeout: 240 + with_items: '{{ play_hosts }}' + connection: local +... diff --git a/tox.ini b/tox.ini index 307d1ffc..05b990f8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,22 +1,19 @@ [tox] -envlist = pep8,py35,py36,bandit,docs +envlist = pep8,py38,bandit,docs [testenv] -basepython=python3 -passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY +pass_env = + http_proxy + HTTP_PROXY + https_proxy + HTTPS_PROXY + no_proxy + NO_PROXY [testenv:venv] commands = {posargs} -[testenv:py35] -setenv = - PYTHONWARNING=all -deps = -r{toxinidir}/requirements-frozen.txt - -r{toxinidir}/test-requirements.txt -commands = - pytest {posargs} - -[testenv:py36] +[testenv:py38] setenv = PYTHONWARNING=all deps = -r{toxinidir}/requirements-frozen.txt @@ -31,7 +28,9 @@ commands = bandit -r promenade [testenv:docs] -whitelist_externals = rm +pass_env = {[pkgenv]pass_env} +allowlist_externals = + rm deps = -r{toxinidir}/doc/requirements.txt commands = @@ -47,7 +46,7 @@ commands = [testenv:freeze] deps = -r{toxinidir}/requirements-direct.txt recreate = True -whitelist_externals = sh +allowlist_externals = sh grep commands= sh -c "pip freeze | grep -vE '^(promenade)|(pkg-resources)' > {toxinidir}/requirements-frozen.txt" @@ -55,7 +54,7 @@ commands= [testenv:gate-lint] deps = jsonschema==2.6.0 -whitelist_externals = sh +allowlist_externals = sh commands = {toxinidir}/tools/lint_gate.sh @@ -66,3 +65,15 @@ commands = yapf -rd {toxinidir}/promenade {toxinidir}/tests {toxinidir}/tools/image_tags.py flake8 {toxinidir}/promenade bandit -r promenade + +[flake8] +# [H106] Don't put vim configuration in source files. +# [H203] Use assertIs(Not)None to check for None. +# [H204] Use assert(Not)Equal to check for equality. +# [H205] Use assert(Greater|Less)(Equal) for comparison. +# [H210] Require 'autospec', 'spec', or 'spec_set' in mock.patch/mock.patch.object calls +# [H904] Delay string interpolations at logging calls. +enable-extensions = H106,H203,H204,H205,H210 +# [E731] Do not assign a lambda expression, use a def. This reduces readability in some cases. +ignore = E126,E731,F405,H102,H301,H306,H401,H404,H405,H904,W503,W504 +exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc,alembic/versions diff --git a/.zuul.yaml b/zuul.d/jobs.yaml similarity index 73% rename from .zuul.yaml rename to zuul.d/jobs.yaml index 94aa6955..a9f56ca0 100644 --- a/.zuul.yaml +++ b/zuul.d/jobs.yaml @@ -1,3 +1,6 @@ +--- +# Copyright 2018 SUSE LINUX GmbH. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -10,61 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -- project: - templates: - - docs-on-readthedocs - vars: - rtd_webhook_id: '38575' - rtd_project_name: 'airship-promenade' - check: - jobs: - - openstack-tox-pep8-bionic - - airship-promenade-lint-ws - - airship-promenade-docker-build-gate-ubuntu_xenial - - airship-promenade-docker-build-gate-ubuntu_bionic - - airship-promenade-chart-build-gate - - airship-promenade-chart-build-latest-htk - - airship-promenade-unit-py35 - - airship-promenade-genesis-containerd-gate - - gate: - jobs: - - openstack-tox-pep8-bionic - - airship-promenade-lint-ws - - airship-promenade-docker-build-gate-ubuntu_xenial - - airship-promenade-docker-build-gate-ubuntu_bionic - - airship-promenade-chart-build-gate - - airship-promenade-unit-py35 - - airship-promenade-genesis-containerd-gate - - post: - jobs: - - airship-promenade-docker-build-post-ubuntu_xenial - - airship-promenade-docker-build-post-ubuntu_bionic - - promenade-upload-git-mirror - -- nodeset: - name: airship-promenade-single-node - nodes: - - name: primary - label: ubuntu-bionic -- nodeset: - name: airship-promenade-single-node-bionic - nodes: - - name: primary - label: ubuntu-bionic - - job: name: airship-promenade-genesis-containerd-gate description: | Deploy airship promenade genesis with containerd - run: tools/zuul/playbooks/deploy-promenade-containerd.yaml + run: + - tools/zuul/playbooks/deploy-promenade-containerd.yaml required-projects: - openstack/openstack-helm-infra timeout: 3600 - nodeset: airship-promenade-single-node-bionic + nodeset: airship-promenade-single-node-focal vars: - distro: ubuntu_bionic + distro: ubuntu_focal - job: name: airship-promenade-lint-ws @@ -72,7 +32,7 @@ Lints all files for trailing whitespace run: tools/zuul/playbooks/zuul-linter.yaml timeout: 300 - nodeset: airship-promenade-single-node + nodeset: airship-promenade-single-node-focal - job: name: airship-promenade-chart-build-gate @@ -80,7 +40,7 @@ Lints charts using pinned HTK run: tools/zuul/playbooks/helm-linter.yaml timeout: 300 - nodeset: airship-promenade-single-node + nodeset: airship-promenade-single-node-focal - job: name: airship-promenade-chart-build-latest-htk @@ -89,33 +49,33 @@ voting: false run: tools/zuul/playbooks/helm-linter.yaml timeout: 300 - nodeset: airship-promenade-single-node + nodeset: airship-promenade-single-node-focal vars: HTK_COMMIT: master - job: - name: airship-promenade-unit-py35 + name: airship-promenade-unit-py38 description: | - Executes unit tests under Python 3.5 + Executes unit tests under Python 3.8 run: tools/zuul/playbooks/make-tests.yaml timeout: 1500 - nodeset: airship-promenade-single-node + nodeset: airship-promenade-single-node-focal files: - ^.*\.py$ - job: name: airship-promenade-linter run: tools/zuul/playbooks/zuul-linter.yaml - nodeset: airship-promenade-single-node + nodeset: airship-promenade-single-node-focal - job: - name: airship-promenade-docker-build-gate-ubuntu_bionic - voting: false + name: airship-promenade-docker-build-gate + voting: true run: tools/zuul/playbooks/docker-image-build.yaml - nodeset: airship-promenade-single-node + nodeset: airship-promenade-single-node-focal vars: publish: false - distro: ubuntu_bionic + distro: ubuntu_focal tags: dynamic: patch_set: true @@ -125,50 +85,15 @@ - ^tests/.*$ - ^tools/.*$ - job: - name: airship-promenade-docker-build-gate-ubuntu_bionic - run: tools/zuul/playbooks/docker-image-build.yaml - nodeset: airship-promenade-single-node - vars: - publish: false - distro: ubuntu_bionic - tags: - dynamic: - patch_set: true - irrelevant-files: - - ^charts/.*$ - - ^etc/.*$ - - ^tests/.*$ - - ^tools/.*$ -- job: - name: airship-promenade-docker-build-post-ubuntu_xenial + name: airship-promenade-docker-build-post voting: false run: tools/zuul/playbooks/docker-image-build.yaml - nodeset: airship-promenade-single-node + nodeset: airship-promenade-single-node-focal secrets: - airship_promenade_quay_creds vars: publish: true - distro: ubuntu_xenial - tags: - dynamic: - branch: true - commit: true - static: - - latest - irrelevant-files: - - ^charts/.*$ - - ^etc/.*$ - - ^tests/.*$ - - ^tools/.*$ -- job: - name: airship-promenade-docker-build-post-ubuntu_bionic - run: tools/zuul/playbooks/docker-image-build.yaml - nodeset: airship-promenade-single-node - secrets: - - airship_promenade_quay_creds - vars: - publish: true - distro: ubuntu_bionic + distro: ubuntu_focal tags: dynamic: branch: true diff --git a/zuul.d/nodesets.yaml b/zuul.d/nodesets.yaml new file mode 100644 index 00000000..e2bbe6ac --- /dev/null +++ b/zuul.d/nodesets.yaml @@ -0,0 +1,22 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- nodeset: + name: airship-promenade-single-node-focal + nodes: + - name: primary + label: ubuntu-focal + groups: + - name: primary + nodes: + - primary \ No newline at end of file diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml new file mode 100644 index 00000000..b26c04f5 --- /dev/null +++ b/zuul.d/project.yaml @@ -0,0 +1,44 @@ +--- +# Copyright 2018 SUSE LINUX GmbH. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- project: + templates: + - docs-on-readthedocs + vars: + rtd_webhook_id: '38575' + rtd_project_name: 'airship-promenade' + check: + jobs: + - openstack-tox-pep8 + # - openstack-tox-py38 + - airship-promenade-lint-ws + - airship-promenade-docker-build-gate + - airship-promenade-chart-build-gate + - airship-promenade-chart-build-latest-htk + - airship-promenade-unit-py38 + - airship-promenade-genesis-containerd-gate + gate: + jobs: + - openstack-tox-pep8 + - airship-promenade-lint-ws + - airship-promenade-docker-build-gate + - airship-promenade-chart-build-gate + - airship-promenade-unit-py38 + - airship-promenade-genesis-containerd-gate + post: + jobs: + - airship-promenade-docker-build-post + - promenade-upload-git-mirror +...