Add Airskiff deployment gate

This PS adds Airskiff deployment gate that tests new armada-go image. Change-Id: Ia0efb71a6a4bc75d884d356509efb7bbdfb8341c
2024-03-19 20:58:31 +00:00 · 2024-03-19 20:58:31 +00:00 · ff3cf5614a
parent 37c26294f0
commit ff3cf5614a
11 changed files with 516 additions and 0 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -14,6 +14,7 @@
    check:
      jobs:
        - armada-go-docker-build-gate-ubuntu_focal
+        - armada-go-airskiff-deployment-focal

    gate:
      jobs:
@ -30,6 +31,97 @@
      - name: primary
        label: ubuntu-focal

+- job:
+    name: armada-go-base
+    abstract: true
+    roles:
+      - zuul: openstack/openstack-helm-infra
+      - zuul: zuul/zuul-jobs
+    required-projects:
+      - name: openstack/openstack-helm
+      - name: openstack/openstack-helm-infra
+      - name: airship/treasuremap
+        override-checkout: v1.9
+    irrelevant-files: &irrelevant-files
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^setup.cfg$
+      - ^deckhand/tests/unit/.*$
+    timeout: 10800
+    pre-run:
+      - tools/gate/playbooks/prepare-hosts.yaml
+      - tools/gate/playbooks/mount-volumes.yaml
+      - tools/gate/playbooks/deploy-env.yaml
+    run:
+      - tools/gate/playbooks/airship-run-scripts.yaml
+    post-run:
+      - tools/gate/playbooks/osh-infra-collect-logs.yaml
+    vars:
+      extra_volume:
+        size: 80G
+        type: Linux
+        mount_point: /opt/ext_vol
+      docker:
+        root_path: "/opt/ext_vol/docker"
+      containerd:
+        root_path: "/opt/ext_vol/containerd"
+      kubeadm:
+        pod_network_cidr: "10.244.0.0/24"
+        service_cidr: "10.96.0.0/16"
+      loopback_setup: true
+      loopback_device: /dev/loop100
+      loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
+      ceph_osd_data_device: /dev/loop100
+      kube_version_repo: "v1.29"
+      # the list of k8s package versions are available here
+      # https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Packages
+      kube_version: "1.29.2-1.1"
+      calico_version: "v3.27.0"
+      coredns_version: "v1.11.1"
+      helm_version: "v3.13.2"
+      yq_version: "v4.6.0"
+      crictl_version: "v1.26.1"
+      zuul_osh_relative_path: ../../openstack/openstack-helm
+      zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
+      zuul_treasuremap_relative_path: ../../airship/treasuremap
+      gate_scripts_relative_path: ../../openstack/openstack-helm-infra
+      run_helm_tests: "no"
+
+- job:
+    name: armada-go-airskiff-deployment-focal
+    nodeset: treasuremap-airskiff-1node-ubuntu_focal
+    description: |
+      Deploy Memcached using Airskiff and submitted Armada-go changes.
+    parent: armada-go-base
+    vars:
+      site: airskiff
+      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+      CLONE_ARMADA_GO: false
+      DISTRO: ubuntu_focal
+      DOCKER_REGISTRY: localhost:5000
+      MAKE_ARMADA_OPERATOR_IMAGES: true
+      USE_ARMADA_GO: true
+      gate_scripts_relative_path: ../../airship/treasuremap
+      gate_scripts:
+        - ./tools/deployment/airskiff/developer/000-prepare-k8s.sh
+        - ./tools/deployment/airskiff/developer/009-setup-apparmor.sh
+        - ./tools/deployment/airskiff/developer/000-clone-dependencies.sh
+        - ./tools/deployment/airskiff/developer/020-setup-client.sh
+        - ./tools/deployment/airskiff/developer/015-make-all-charts.sh
+        - ./tools/deployment/airskiff/developer/017-make-all-images.sh
+        - ./tools/deployment/airskiff/developer/025-start-artifactory.sh
+        - ./tools/deployment/airskiff/developer/026-reduce-site.sh
+        - ./tools/deployment/airskiff/developer/027-enable-armada-operator.sh
+        - ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
+        - ./tools/deployment/airskiff/developer/100-deploy-osh.sh
+        - ./tools/deployment/airskiff/common/os-env.sh
+        - ./tools/gate/wait-for-shipyard.sh
+    irrelevant-files: *irrelevant-files

 - job:
    name: armada-go-docker-build-gate-ubuntu_focal
--- a/tools/gate/playbooks/airship-run-scripts.yaml
+++ b/tools/gate/playbooks/airship-run-scripts.yaml
@ -0,0 +1,75 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: primary
+  tasks:
+    - name: Override images
+      when: buildset_registry is defined
+      vars:
+        work_dir: "{{ zuul.project.src_dir }}"
+      block:
+        - name: Buildset registry alias
+          include_role:
+            name: deploy-env
+            tasks_from: buildset_registry_alias
+
+        - name: Print zuul
+          debug:
+            var: zuul
+
+        - name: Override proposed images from artifacts
+          shell: >
+            find {{ override_paths | join(" ") }} -type f -exec sed -Ei
+            "s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} +
+          loop: "{{ zuul.artifacts | default([]) }}"
+          args:
+            chdir: "{{ work_dir }}"
+          loop_control:
+            loop_var: zj_zuul_artifact
+          when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'"
+          vars:
+            tag: "{{ zj_zuul_artifact.metadata.tag }}"
+            repo: "{{ zj_zuul_artifact.metadata.repository }}"
+            override_paths:
+              - ../openstack-helm*/*/values*
+              - ../openstack-helm-infra/tools/deployment/
+
+        - name: Diff
+          shell: |
+              set -ex;
+              for dir in openstack-helm openstack-helm-infra; do
+                path="{{ work_dir }}/../${dir}/"
+                if [ ! -d "${path}" ]; then continue; fi
+                echo "${dir} diff"
+                cd "${path}"; git diff; cd -;
+              done
+
+    - name: "creating directory for run artifacts"
+      file:
+        path: "/tmp/artifacts"
+        state: directory
+
+    - name: Run gate scripts
+      include_role:
+        name: "{{ ([item] | flatten | length == 1) | ternary('airship-run-script', 'airship-run-script-set') }}"
+      vars:
+        workload: "{{ [item] | flatten }}"
+      loop: "{{ gate_scripts }}"
+
+    - name: "Downloads artifacts to executor"
+      synchronize:
+        src: "/tmp/artifacts"
+        dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+        mode: pull
+      ignore_errors: True
+...
--- a/tools/gate/playbooks/deploy-env.yaml
+++ b/tools/gate/playbooks/deploy-env.yaml
@ -0,0 +1,36 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  become: true
+  gather_facts: true
+
+  roles:
+    - ensure-python
+    - ensure-pip
+    - ensure-tox
+    - clear-firewall
+    - deploy-apparmor
+    - deploy-selenium
+    - deploy-env
+
+  tasks:
+    - name: Install Packaging python module for tools/airship
+      block:
+        - pip:
+            name: packaging
+            version: 23.1
+            executable: pip3
+      become: True
+
+...
--- a/tools/gate/playbooks/mount-volumes.yaml
+++ b/tools/gate/playbooks/mount-volumes.yaml
@ -0,0 +1,17 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  roles:
+    - mount-extra-volume
+...
--- a/tools/gate/playbooks/osh-infra-collect-logs.yaml
+++ b/tools/gate/playbooks/osh-infra-collect-logs.yaml
@ -0,0 +1,43 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+    logs_dir: "/tmp/logs"
+  roles:
+    - gather-host-logs
+  tags:
+    - gather-host-logs
+
+- hosts: primary
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+    logs_dir: "/tmp/logs"
+  roles:
+    - helm-release-status
+    - describe-kubernetes-objects
+    - gather-pod-logs
+    - gather-prom-metrics
+    - gather-selenium-data
+  tags:
+    - helm-release-status
+    - describe-kubernetes-objects
+    - gather-pod-logs
+    - gather-prom-metrics
+    - gather-selenium-data
+...
--- a/tools/gate/playbooks/prepare-hosts.yaml
+++ b/tools/gate/playbooks/prepare-hosts.yaml
@ -0,0 +1,17 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  roles:
+    - start-zuul-console
+...
--- a/tools/gate/playbooks/roles
+++ b/tools/gate/playbooks/roles
@ -0,0 +1 @@
+../roles/
--- a/tools/gate/roles/airship-run-script-set/defaults/main.yaml
+++ b/tools/gate/roles/airship-run-script-set/defaults/main.yaml
@ -0,0 +1,46 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+ceph_osd_data_device: "/dev/loop0"
+kubeadm:
+  pod_network_cidr: "10.244.0.0/24"
+osh_params:
+  container_distro_name: ubuntu
+  container_distro_version: focal
+  # feature_gates:
+site: airskiff
+HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+COREDNS_VERSION: v1.11.1
+DISTRO: ubuntu_focal
+DOCKER_REGISTRY: quay.io
+CLONE_ARMADA: true
+CLONE_ARMADA_GO: true
+CLONE_ARMADA_OPERATOR: true
+CLONE_DECKHAND: true
+CLONE_SHIPYARD: true
+CLONE_PORTHOLE: true
+CLONE_PROMENADE: true
+CLONE_MAAS: true
+CLONE_OSH: true
+MAKE_ARMADA_IMAGES: false
+MAKE_ARMADA_GO_IMAGES: false
+MAKE_ARMADA_OPERATOR_IMAGES: false
+MAKE_DECKHAND_IMAGES: false
+MAKE_SHIPYARD_IMAGES: false
+MAKE_PORTHOLE_IMAGES: false
+MAKE_PROMENADE_IMAGES: false
+USE_ARMADA_GO: false
+...
--- a/tools/gate/roles/airship-run-script-set/tasks/main.yaml
+++ b/tools/gate/roles/airship-run-script-set/tasks/main.yaml
@ -0,0 +1,83 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- block:
+    - name: "Run script set {{ workload }}"
+      shell: |
+        set -xe;
+        {{ gate_script_path }}
+      loop: "{{ workload }}"
+      loop_control:
+        loop_var: gate_script_path
+        pause: 5
+      args:
+        chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}"
+      environment:
+        CEPH_OSD_DATA_DEVICE: "{{ ceph_osd_data_device }}"
+        POD_NETWORK_CIDR: "{{ kubeadm.pod_network_cidr }}"
+        zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}"
+        OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}"
+        OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}"
+        CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}"
+        CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}"
+        FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
+        RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
+        PL_SITE: "{{ site | default('airskiff') }}"
+        HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}"
+        HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+        OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+        OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
+        COREDNS_VERSION: "{{ coredns_version | default('v1.11.1') }}"
+        DISTRO: "{{ DISTRO | default('ubuntu_focal') }}"
+        DOCKER_REGISTRY: "{{ DOCKER_REGISTRY | default('quay.io') }}"
+        CLONE_ARMADA: "{{ CLONE_ARMADA | default('true') }}"
+        CLONE_ARMADA_GO: "{{ CLONE_ARMADA_GO | default('true') }}"
+        CLONE_ARMADA_OPERATOR: "{{ CLONE_ARMADA_OPERATOR | default('true') }}"
+        CLONE_DECKHAND: "{{ CLONE_DECKHAND | default('true') }}"
+        CLONE_SHIPYARD: "{{ CLONE_SHIPYARD | default('true') }}"
+        CLONE_PORTHOLE: "{{ CLONE_PORTHOLE | default('true') }}"
+        CLONE_PROMENADE: "{{ CLONE_PROMENADE | default('true') }}"
+        CLONE_MAAS: "{{ CLONE_MAAS | default('true') }}"
+        CLONE_OSH: "{{ CLONE_OSH | default('true') }}"
+        MAKE_ARMADA_IMAGES: "{{ MAKE_ARMADA_IMAGES | default('false') }}"
+        MAKE_ARMADA_GO_IMAGES: "{{ MAKE_ARMADA_GO_IMAGES | default('false') }}"
+        MAKE_ARMADA_OPERATOR_IMAGES: "{{ MAKE_ARMADA_OPERATOR_IMAGES | default('false') }}"
+        MAKE_DECKHAND_IMAGES: "{{ MAKE_DECKHAND_IMAGES | default('false') }}"
+        MAKE_SHIPYARD_IMAGES: "{{ MAKE_SHIPYARD_IMAGES | default('false') }}"
+        MAKE_PORTHOLE_IMAGES: "{{ MAKE_PORTHOLE_IMAGES | default('false') }}"
+        MAKE_PROMENADE_IMAGES: "{{ MAKE_PROMENADE_IMAGES | default('false') }}"
+        USE_ARMADA_GO: "{{ USE_ARMADA_GO | default('false') }}"
+      # NOTE(aostapenko) using bigger than async_status timeout due to async_status issue with
+      # not recognizing timed out jobs: https://github.com/ansible/ansible/issues/25637
+      async: 3600
+      poll: 0
+      register: async_results
+
+    - name: Wait for script set to finish
+      async_status:
+        jid: '{{ item.ansible_job_id }}'
+      register: jobs
+      until: jobs.finished
+      delay: 5
+      retries: 360
+      loop: "{{ async_results.results }}"
+
+  always:
+    - name: Print script set output
+      shell: |
+          # NOTE(aostapenko) safely retrieving items for the unlikely case if jobs timed out in async_status
+          echo 'STDOUT:\n{{ item.get("stdout") | regex_replace("\'", "") }}\nSTDERR:\n{{ item.get("stderr") | regex_replace("\'", "") }}'
+      loop: "{{ jobs.results }}"
+...
--- a/tools/gate/roles/airship-run-script/defaults/main.yaml
+++ b/tools/gate/roles/airship-run-script/defaults/main.yaml
@ -0,0 +1,47 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+ceph_osd_data_device: "/dev/loop0"
+kubeadm:
+  pod_network_cidr: "10.244.0.0/24"
+osh_params:
+  container_distro_name: ubuntu
+  container_distro_version: focal
+  # feature_gates:
+site: airskiff
+HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+COREDNS_VERSION: v1.11.1
+DISTRO: ubuntu_focal
+DOCKER_REGISTRY: quay.io
+
+CLONE_ARMADA: true
+CLONE_ARMADA_GO: true
+CLONE_ARMADA_OPERATOR: true
+CLONE_DECKHAND: true
+CLONE_SHIPYARD: true
+CLONE_PORTHOLE: true
+CLONE_PROMENADE: true
+CLONE_MAAS: true
+CLONE_OSH: true
+MAKE_ARMADA_IMAGES: false
+MAKE_ARMADA_GO_IMAGES: false
+MAKE_ARMADA_OPERATOR_IMAGES: false
+MAKE_DECKHAND_IMAGES: false
+MAKE_SHIPYARD_IMAGES: false
+MAKE_PORTHOLE_IMAGES: false
+MAKE_PROMENADE_IMAGES: false
+USE_ARMADA_GO: false
+...
--- a/tools/gate/roles/airship-run-script/tasks/main.yaml
+++ b/tools/gate/roles/airship-run-script/tasks/main.yaml
@ -0,0 +1,59 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: "Run script {{ workload[0] }}"
+  shell: |
+    set -xe;
+    {{ gate_script_path }}
+  vars:
+    gate_script_path: "{{ workload[0] }}"
+  args:
+    chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}"
+  environment:
+    CEPH_OSD_DATA_DEVICE: "{{ ceph_osd_data_device }}"
+    POD_NETWORK_CIDR: "{{ kubeadm.pod_network_cidr }}"
+    zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+    OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+    OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}"
+    OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}"
+    OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}"
+    CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}"
+    CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}"
+    FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
+    RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
+    PL_SITE: "{{ site | default('airskiff') }}"
+    HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}"
+    HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+    OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+    OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
+    COREDNS_VERSION: "{{ coredns_version | default('v1.11.1') }}"
+    DISTRO: "{{ DISTRO | default('ubuntu_focal') }}"
+    DOCKER_REGISTRY: "{{ DOCKER_REGISTRY | default('quay.io') }}"
+    CLONE_ARMADA: "{{ CLONE_ARMADA | default('true') }}"
+    CLONE_ARMADA_GO: "{{ CLONE_ARMADA_GO | default('true') }}"
+    CLONE_ARMADA_OPERATOR: "{{ CLONE_ARMADA_OPERATOR | default('true') }}"
+    CLONE_DECKHAND: "{{ CLONE_DECKHAND | default('true') }}"
+    CLONE_SHIPYARD: "{{ CLONE_SHIPYARD | default('true') }}"
+    CLONE_PORTHOLE: "{{ CLONE_PORTHOLE | default('true') }}"
+    CLONE_PROMENADE: "{{ CLONE_PROMENADE | default('true') }}"
+    CLONE_MAAS: "{{ CLONE_MAAS | default('true') }}"
+    CLONE_OSH: "{{ CLONE_OSH | default('true') }}"
+    MAKE_ARMADA_IMAGES: "{{ MAKE_ARMADA_IMAGES | default('false') }}"
+    MAKE_ARMADA_GO_IMAGES: "{{ MAKE_ARMADA_GO_IMAGES | default('false') }}"
+    MAKE_ARMADA_OPERATOR_IMAGES: "{{ MAKE_ARMADA_OPERATOR_IMAGES | default('false') }}"
+    MAKE_DECKHAND_IMAGES: "{{ MAKE_DECKHAND_IMAGES | default('false') }}"
+    MAKE_SHIPYARD_IMAGES: "{{ MAKE_SHIPYARD_IMAGES | default('false') }}"
+    MAKE_PORTHOLE_IMAGES: "{{ MAKE_PORTHOLE_IMAGES | default('false') }}"
+    MAKE_PROMENADE_IMAGES: "{{ MAKE_PROMENADE_IMAGES | default('false') }}"
+    USE_ARMADA_GO: "{{ USE_ARMADA_GO | default('false') }}"
+...