Use deploy-env role

Change-Id: I50530eabc8506319478d6ed02b7abb5eb1d2dd05
This commit is contained in:
Sergiy Markin 2024-02-16 01:04:08 +00:00
parent 5b7da9ebd6
commit 41b673691b
15 changed files with 275 additions and 202 deletions

View File

@ -78,36 +78,35 @@
name: deckhand-openstack-tox-pep8-focal
parent: openstack-tox-pep8
description: Runs pep8 job on focal
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
- job:
name: deckhand-openstack-tox-cover-focal
parent: openstack-tox-cover
description: Runs cover job on focal
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
- job:
name: deckhand-tox-py38-postgresql
parent: openstack-tox-py38
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
pre-run:
- tools/gate/playbooks/install-postgresql.yaml
vars:
tox_envlist: py38-postgresql
- job:
name: deckhand-functional-uwsgi-py38
voting: true
description: |
Run tox-based functional tests for the Airship Deckhand project using a
minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
for ephemeral PostgreSQL DB, under cPython version 3.8.
run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml
nodeset: deckhand-single-node-focal
vars:
tox_envlist: py38-functional-dev
distro: ubuntu_focal
name: deckhand-base
abstract: true
roles:
- zuul: openstack/openstack-helm-infra
- zuul: aprship/porthole
- zuul: zuul/zuul-jobs
required-projects:
- name: openstack/openstack-helm
- name: openstack/openstack-helm-infra
- name: airship/treasuremap
override-checkout: v1.9
irrelevant-files: &irrelevant-files
- ^.*\.rst$
- ^doc/.*$
@ -115,22 +114,43 @@
- ^releasenotes/.*$
- ^setup.cfg$
- ^deckhand/tests/unit/.*$
- job:
name: deckhand-functional-docker-base
description: |
Base job for running deckhand functional tests. Runs tests
against Docker image generated from source code.
nodeset: deckhand-single-node-focal
roles:
- zuul: openstack/openstack-helm-infra
timeout: 3600
timeout: 10800
pre-run:
- tools/gate/playbooks/osh-infra-upgrade-host.yaml
- tools/gate/playbooks/osh-infra-deploy-docker.yaml
run: tools/gate/playbooks/run-functional-tests-docker.yaml
required-projects:
- openstack/openstack-helm-infra
- tools/gate/playbooks/prepare-hosts.yaml
- tools/gate/playbooks/mount-volumes.yaml
- tools/gate/playbooks/deploy-env.yaml
run:
- tools/gate/playbooks/airship-run-scripts.yaml
post-run:
- tools/gate/playbooks/osh-infra-collect-logs.yaml
vars:
extra_volume:
size: 80G
type: Linux
mount_point: /opt/ext_vol
docker:
root_path: "/opt/ext_vol/docker"
containerd:
root_path: "/opt/ext_vol/containerd"
kubeadm:
pod_network_cidr: "10.244.0.0/24"
service_cidr: "10.96.0.0/16"
loopback_setup: true
loopback_device: /dev/loop100
loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
ceph_osd_data_device: /dev/loop100
kube_version: "1.28.4-1.1"
calico_version: "v3.27.0"
coredns_version: "v1.11.1"
helm_version: "v3.13.2"
yq_version: "v4.6.0"
crictl_version: "v1.26.1"
zuul_osh_relative_path: ../../openstack/openstack-helm
zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
zuul_treasuremap_relative_path: ../../airship/treasuremap
gate_scripts_relative_path: ../../openstack/openstack-helm-infra
run_helm_tests: "no"
- job:
name: deckhand-functional-docker-py38
@ -138,39 +158,67 @@
Run tox-based functional tests for the Airship Deckhand project under
cPython version 3.8. Uses tox with the ``functional-py38`` environment.
Ubuntu (focal) image is built and used.
parent: deckhand-functional-docker-base
nodeset: deckhand-single-node-focal
parent: deckhand-base
run: tools/gate/playbooks/run-functional-tests-docker.yaml
nodeset: treasuremap-airskiff-1node-ubuntu_focal
vars:
tox_envlist: py38-functional
disable_keystone: true
distro: ubuntu_focal
irrelevant-files: *irrelevant-files
- job:
name: deckhand-functional-uwsgi-py38
description: |
Run tox-based functional tests for the Airship Deckhand project using a
minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
for ephemeral PostgreSQL DB, under cPython version 3.8.
parent: deckhand-base
run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml
nodeset: treasuremap-airskiff-1node-ubuntu_focal
vars:
tox_envlist: py38-functional-dev
distro: ubuntu_focal
irrelevant-files: *irrelevant-files
- job:
name: deckhand-integration-uwsgi-py38
voting: true
description: |
Run tox-based integration tests for the Airship Deckhand project using a
minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
for ephemeral PostgreSQL DB, under cPython version 3.8.
timeout: 9600
parent: deckhand-base
run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
irrelevant-files: *irrelevant-files
vars:
disable_keystone: true
distro: ubuntu_focal
- job:
name: deckhand-integration-docker-py38
description: |
Run tox-based integration tests for the Airship Deckhand project under
cPython version 3.8. Builds ubuntu (focal) deckhand image.
parent: deckhand-base
run: tools/gate/playbooks/run-integration-tests-docker.yaml
nodeset: treasuremap-airskiff-1node-ubuntu_focal
vars:
tox_envlist: py38-functional
disable_keystone: true
distro: ubuntu_focal
irrelevant-files: *irrelevant-files
- job:
name: deckhand-chart-build-gate
description: |
Build charts using pinned Helm toolkit.
timeout: 900
run: tools/gate/playbooks/build-charts.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
vars:
HTK_COMMIT: ae91cf3fc3f288b6d92ace4a3a405606a653638f
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
- job:
name: deckhand-chart-build-latest-htk
@ -178,53 +226,14 @@
Build charts using latest Helm toolkit.
timeout: 900
run: tools/gate/playbooks/build-charts.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
vars:
HTK_COMMIT: master
- job:
name: deckhand-integration-docker-base
description: |
Base job for running deckhand integration tests. Runs tests
against Docker image generated from source code.
nodeset: deckhand-single-node-focal
timeout: 3600
roles:
- zuul: openstack/openstack-helm-infra
vars:
zuul_osh_relative_path: ../../openstack/openstack-helm/
zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra/
zuul_treasuremap_relative_path: ../../airship/treasuremap/
pre-run:
- tools/gate/playbooks/osh-infra-upgrade-host.yaml
- tools/gate/playbooks/osh-infra-deploy-docker.yaml
run:
- tools/gate/playbooks/run-integration-tests-docker.yaml
post-run: tools/gate/playbooks/osh-infra-collect-logs.yaml
required-projects:
- openstack/openstack-helm
- openstack/openstack-helm-infra
- airship/treasuremap
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
- job:
name: deckhand-integration-docker-py38
description: |
Run tox-based integration tests for the Airship Deckhand project under
cPython version 3.8. Builds ubuntu (focal) deckhand image.
parent: deckhand-integration-docker-base
nodeset: deckhand-single-node-focal
vars:
disable_keystone: false
distro: ubuntu_focal
- job:
name: deckhand-airskiff-deployment-focal
voting: true
nodeset: deckhand-single-node-airskiff-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
description: |
Deploy Memcached using Airskiff and submitted Deckhand changes.
timeout: 9600
@ -243,17 +252,14 @@
OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
distro: ubuntu_focal
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
irrelevant-files: *irrelevant-files
- job:
name: deckhand-docker-build-gate
timeout: 3600
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
irrelevant-files: &non-code-files-template
- ^.*\.rst$
- ^doc/.*$
@ -276,7 +282,7 @@
listed in vars section. Waits in Zuul queue for a node (VM) assignment.
timeout: 3600
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
secrets:
- airship_deckhand_quay_creds
irrelevant-files: *non-code-files-template

View File

@ -0,0 +1,75 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: primary
tasks:
- name: Override images
when: buildset_registry is defined
vars:
work_dir: "{{ zuul.project.src_dir }}"
block:
- name: Buildset registry alias
include_role:
name: deploy-env
tasks_from: buildset_registry_alias
- name: Print zuul
debug:
var: zuul
- name: Override proposed images from artifacts
shell: >
find {{ override_paths | join(" ") }} -type f -exec sed -Ei
"s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} +
loop: "{{ zuul.artifacts | default([]) }}"
args:
chdir: "{{ work_dir }}"
loop_control:
loop_var: zj_zuul_artifact
when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'"
vars:
tag: "{{ zj_zuul_artifact.metadata.tag }}"
repo: "{{ zj_zuul_artifact.metadata.repository }}"
override_paths:
- ../openstack-helm*/*/values*
- ../openstack-helm-infra/tools/deployment/
- name: Diff
shell: |
set -ex;
for dir in openstack-helm openstack-helm-infra; do
path="{{ work_dir }}/../${dir}/"
if [ ! -d "${path}" ]; then continue; fi
echo "${dir} diff"
cd "${path}"; git diff; cd -;
done
- name: "creating directory for run artifacts"
file:
path: "/tmp/artifacts"
state: directory
- name: Run gate scripts
include_role:
name: "{{ ([item] | flatten | length == 1) | ternary('airship-run-script', 'airship-run-script-set') }}"
vars:
workload: "{{ [item] | flatten }}"
loop: "{{ gate_scripts }}"
- name: "Downloads artifacts to executor"
synchronize:
src: "/tmp/artifacts"
dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
mode: pull
ignore_errors: True
...

View File

@ -0,0 +1,36 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: all
become: true
gather_facts: true
roles:
- ensure-python
- ensure-pip
- ensure-tox
- clear-firewall
- deploy-apparmor
- deploy-selenium
- deploy-env
tasks:
- name: Install Packaging python module for tools/airship
block:
- pip:
name: packaging
version: 23.1
executable: pip3
become: True
...

View File

@ -0,0 +1,17 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: all
roles:
- mount-extra-volume
...

View File

@ -1,5 +1,3 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -12,17 +10,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: all
vars_files:
- vars.yaml
vars:
work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
logs_dir: "/tmp/logs"
roles:
- gather-host-logs
- helm-release-status
- describe-kubernetes-objects
- gather-pod-logs
tags:
- gather-host-logs
- hosts: primary
vars_files:
- vars.yaml
vars:
work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
logs_dir: "/tmp/logs"
roles:
- helm-release-status
- describe-kubernetes-objects
- gather-pod-logs
- gather-prom-metrics
- gather-selenium-data
tags:
- helm-release-status
- describe-kubernetes-objects
- gather-pod-logs
- gather-prom-metrics
- gather-selenium-data
...

View File

@ -0,0 +1,17 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: all
roles:
- start-zuul-console
...

View File

@ -16,34 +16,10 @@
gather_facts: False
become: yes
roles:
- install-test-requirements
- build-images
tags:
- build-images
- hosts: all
gather_facts: False
roles:
- deploy-postgresql
tags:
- deploy-postgresql
- hosts: all
gather_facts: False
roles:
- generate-test-config
tags:
- generate-test-config
- hosts: all
gather_facts: False
roles:
- deploy-deckhand
tags:
- deploy-deckhand
- hosts: all
gather_facts: False
roles:
- run-functional-tests
tags:
- run-functional-tests

View File

@ -15,14 +15,6 @@
- hosts: all
gather_facts: False
roles:
- bindep
- ensure-docker
- ensure-python
- ensure-pip
- disable-systemd-resolved
- install-postgresql
- install-test-requirements
- run-functional-tests
tags:
- install-postgresql
- run-functional-tests

View File

@ -16,12 +16,10 @@
vars_files:
- vars.yaml
roles:
- clear-firewall
- bindep
- ensure-docker
- ensure-python
- ensure-pip
- disable-systemd-resolved
- role: add-authorized-keys
public_keys:
- public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com
- install-test-requirements
- deploy-keystone-dependencies
- deploy-barbican
@ -34,12 +32,4 @@
- generate-test-config
- deploy-deckhand
- run-integration-tests
tags:
- build-images
- deploy-keystone-dependencies
- deploy-barbican
- build-charts
- deploy-postgresql
- generate-test-config
- deploy-deckhand
- run-integration-tests

View File

@ -13,21 +13,12 @@
# limitations under the License.
- hosts: all
vars_files:
- vars.yaml
gather_facts: False
roles:
- clear-firewall
- bindep
- ensure-docker
- ensure-python
- ensure-pip
- disable-systemd-resolved
- role: add-authorized-keys
public_keys:
- public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com
- install-postgresql
- install-test-requirements
- run-integration-tests
tags:
- install-postgresql
- run-integration-tests

View File

@ -17,24 +17,6 @@
set_fact:
zuul_site_mirror_fqdn: "{{ lookup('env','zuul_site_mirror_fqdn') }}"
- name: Checkout treasuremap v1.9 branch
shell: |
set -xe;
git checkout v1.9
environment:
zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
args:
chdir: "{{ zuul.project.src_dir }}/{{ zuul_treasuremap_relative_path | default('') }}"
- name: Deploy Kubernetes
shell: |
set -xe;
./tools/deployment/airskiff/developer/010-deploy-k8s.sh
environment:
zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
args:
chdir: "{{ zuul.project.src_dir }}/{{ zuul_treasuremap_relative_path | default('') }}"
- name: Make OSH charts
shell: |
set -xe
@ -53,6 +35,12 @@
args:
chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
- name: Prepare kubernetes
shell: |
set -xe;
./tools/deployment/airskiff/developer/000-prepare-k8s.sh
args:
chdir: "{{ zuul.project.src_dir }}/{{ zuul_treasuremap_relative_path | default('') }}"
- name: Setup docker-based openstack client
shell: |
@ -64,7 +52,7 @@
- name: Deploy Ingress
shell: |
set -xe;
./tools/deployment/component/common/ingress.sh
./tools/deployment/common/ingress.sh
args:
chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}"

View File

@ -59,16 +59,6 @@
sudo -E -H pip3 install -r requirements-frozen.txt
echo "$(pwd)" | sudo tee /usr/local/lib/python3.8/dist-packages/deckhand.pth
sudo fdisk --list
df -h
sudo mkdir -p /opt/ext_vol
BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep -E 80G | grep Linux | awk '{print $1}')
if ! mount | grep "${BIG_VOLUME}"
then
sudo mkfs.ext4 "${BIG_VOLUME}"
sudo mount "${BIG_VOLUME}" /opt/ext_vol
df -h
fi
pifpaf run postgresql -- ./tools/integration-tests.sh
args:
chdir: "{{ zuul.project.src_dir }}"

View File

@ -17,7 +17,7 @@
set -x
HELM=$1
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"}
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz"}
function install_helm_binary {

View File

@ -17,7 +17,7 @@
set -eux
HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"d0c93b5648704b9d9999412d932f8329c80362c0"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"cfff60ec10a6c386f38db79bb9f59a552c2b032f"}
TMP_DIR=$(mktemp -d)

View File

@ -56,30 +56,10 @@ function deploy_osh_keystone_barbican {
git clone https://git.openstack.org/openstack/openstack-helm.git ../openstack-helm
fi
if [ ! -d "$TM_PATH" ]; then
git clone https://git.openstack.org/airship/treasuremap.git ../treasuremap
pushd ../treasuremap
git checkout v1.9
popd
fi
cd "${TM_PATH}"
# Deploy required packages
./tools/deployment/airskiff/developer/009-setup-apparmor.sh
#
# Deploy Kubernetes
./tools/deployment/airskiff/developer/010-deploy-k8s.sh
#
# Make charts
./tools/deployment/airskiff/developer/015-make-all-charts.sh
#
# Deploy docker-based openstack client
./tools/deployment/airskiff/developer/020-setup-client.sh
cd "${OSH_PATH}"
# Deploy the ingress controller
./tools/deployment/component/common/ingress.sh
./tools/deployment/common/ingress.sh
# Deploy NFS Provisioner
./tools/deployment/component/nfs-provisioner/nfs-provisioner.sh
# Deploy MariaDB