ETCD remote backup enhancements

1) Include framework for remote etcd backups. 2) Use porthole etcdctl utility image for backups. 3) Move helm-toolkit pin to latest commit. 4) Add a keystone user for RGW. 5) Add a secret for Swift API access. 6) Add a secret for backup/restore configuration. Change-Id: Ica549c3b6bc00ca55540b8ffedd4c46af0d8d25e
2020-05-15 01:57:58 +00:00 · 2020-05-15 01:57:58 +00:00 · 95c1689e03
parent 0f4a0d642f
commit 95c1689e03
13 changed files with 384 additions and 40 deletions
--- a/charts/etcd/templates/bin/_etcdbackup.tpl
+++ b/charts/etcd/templates/bin/_etcdbackup.tpl
@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 {{/*
 Copyright 2017 AT&T Intellectual Property.  All other rights reserved.

@ -14,49 +14,51 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */}}
-set -ex
-BACKUP_DIR="/var/lib/etcd/backup"
+set -x
+BACKUP_DIR={{ .Values.backup.host_backup_path }}
 BACKUP_LOG={{ .Values.backup.backup_log_file | quote }}
 NUM_TO_KEEP={{ .Values.backup.no_backup_keep | quote }}
+REMOTE_BACKUP_DAYS_TO_KEEP={{ .Values.backup.remote_backup.days_to_keep | quote }}
 BACKUP_FILE_NAME={{ .Values.service.name | quote }}
 SKIP_BACKUP=0

-etcdbackup() {
-  etcdctl snapshot save $BACKUP_DIR/$BACKUP_FILE_NAME-backup-$(date +"%m-%d-%Y-%H-%M-%S").db >> $BACKUP_LOG
+source /tmp/bin/backup_main.sh
+
+# Export the variables required by the framework
+#  Note: REMOTE_BACKUP_ENABLED and CONTAINER_NAME are already exported
+export DB_NAMESPACE=${POD_NAMESPACE}
+export DB_NAME="etcd"
+export LOCAL_DAYS_TO_KEEP=$NUM_TO_KEEP
+export REMOTE_DAYS_TO_KEEP=$REMOTE_BACKUP_DAYS_TO_KEEP
+export ARCHIVE_DIR=${BACKUP_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive
+
+dump_databases_to_directory() {
+  TMP_DIR=$1
+  LOG_FILE=${2:-BACKUP_LOG}
+
+  cd $TMP_DIR
+  etcdctl snapshot save --command-timeout=5m $TMP_DIR/$BACKUP_FILE_NAME.$DB_NAMESPACE.all.db >> $LOG_FILE
  BACKUP_RETURN_CODE=$?
  if [[ $BACKUP_RETURN_CODE != 0 ]]; then
-    echo "There was an error backing up the databases. Return code was $BACKUP_RETURN_CODE."
+    log ERROR $DB_NAME "There was an error backing up the databases." $LOG_FILE
    exit $BACKUP_RETURN_CODE
  fi
-  LATEST_BACKUP=`ls -t1 $BACKUP_DIR | grep $BACKUP_FILE_NAME | head -1`
-  echo "Archiving $LATEST_BACKUP..."
-  cd $BACKUP_DIR
-  tar -czf $BACKUP_DIR/$LATEST_BACKUP.tar.gz $LATEST_BACKUP
-  rm -rf $LATEST_BACKUP
-  echo "Clearing earliest backups..."
-  NUM_LOCAL_BACKUPS=`ls -1 $BACKUP_DIR | grep $BACKUP_FILE_NAME | wc -l`
-  while [ $NUM_LOCAL_BACKUPS -gt $NUM_TO_KEEP ]
-  do
-    EARLIEST_BACKUP=`ls -tr1 $BACKUP_DIR | grep $BACKUP_FILE_NAME | head -1`
-    echo "Deleting $EARLIEST_BACKUP..."
-    rm -rf "$BACKUP_DIR/$EARLIEST_BACKUP"
-    NUM_LOCAL_BACKUPS=`ls -1 $BACKUP_DIR | grep $BACKUP_FILE_NAME | wc -l`
-  done
 }

 if ! [ -x "$(which etcdctl)" ]; then
-  echo "ERROR: etcdctl not available, Please use the correct image."
+  log ERROR $DB_NAME "etcdctl not available, Please use the correct image." $LOG_FILE
  SKIP_BACKUP=1
 fi

 if [ ! -d "$BACKUP_DIR" ]; then
-  echo "ERROR: $BACKUP_DIR doesn't exist, Backup will not continue"
+  log ERROR $DB_NAME "$BACKUP_DIR doesn't exist, Backup will not continue" $LOG_FILE
  SKIP_BACKUP=1
 fi

 if [ $SKIP_BACKUP -eq 0 ]; then
-  etcdbackup
+  # Call main program to start the database backup
+  backup_databases
 else
-  echo "Error: etcd backup failed."
+  log ERROR $DB_NAME "etcd backup failed." $LOG_FILE
  exit 1
 fi
--- a/charts/etcd/templates/bin/_etcdrestore.tpl
+++ b/charts/etcd/templates/bin/_etcdrestore.tpl
@ -0,0 +1,111 @@
+#!/bin/bash
+
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+set -x
+# Capture the user's command line arguments
+ARGS=("$@")
+
+source /tmp/restore_main.sh
+
+# Export the variables needed by the framework
+export DB_NAME="etcd"
+export DB_NAMESPACE=${POD_NAMESPACE}
+export SINGLE_DB_NAME_DIR=${ETCD_BACKUP_BASE_PATH}/db/${DB_NAMESPACE}/${DB_NAME}/archive
+
+# Extract all databases from an archive and put them in the requested
+# file.
+get_databases() {
+  TMP_DIR=$1
+  DB_FILE=$2
+
+  ETCD_FILE={{ .Values.service.name }}.$POD_NAMESPACE.all.db
+  if [[ -e $TMP_DIR/$ETCD_FILE ]]; then
+    grep 'CREATE DATABASE' $TMP_DIR/$ETCD_FILE | awk '{ print $3 }' > $DB_FILE
+  else
+    # no databases - just touch the file
+    touch $DB_FILE
+  fi
+}
+
+restore_single_db() {
+  SINGLE_DB_NAME=$1
+  TMP_DIR=$2
+  ANCHOR_POD=$SINGLE_DB_NAME
+  if [[ -f $TMP_DIR/$ETCD_FILE ]]; then
+
+       # Check etcd-anchor pod
+       if [[ ! $(kubectl get pods -n $POD_NAMESPACE $ANCHOR_POD) ]]; then
+         echo "Could not find pod $ANCHOR_POD."
+         return 1
+       fi
+
+       # Copy backup to etcd-anchor
+       kubectl cp -n $POD_NAMESPACE $TMP_DIR/$ETCD_FILE $ANCHOR_POD:/
+       if [[ $? -ne 0 ]]; then
+         echo "Could not copy backup to $ANCHOR_POD."
+         return 1
+       fi
+
+       # Node Name
+       NAME=$(kubectl get pods -n $POD_NAMESPACE $ANCHOR_POD -o jsonpath={.spec.nodeName})
+
+       # Initial Cluster
+       INITIAL_CLUSTER="$(etcdctl member list|awk -F , '{gsub (" ", "", $0);printf "%s=%s,", $3,$4}')"
+       INITIAL_ADVERTISE_PEER_URLS=$(kubectl exec -it -n $POD_NAMESPACE $ANCHOR_POD -- env| grep PEER |awk -F = '{print $2}')
+
+       # Restore snapshot
+       kubectl exec -it -n $POD_NAMESPACE $ANCHOR_POD -- env ETCD_FILE=$ETCD_FILE NAME=$NAME INITIAL_CLUSTER=$INITIAL_CLUSTER INITIAL_ADVERTISE_PEER_URLS=$INITIAL_ADVERTISE_PEER_URLS;/usr/local/bin/etcdctl snapshot restore $ETCD_FILE --name $NAME --initial-cluster "$INITIAL_CLUSTER" --initial-cluster-token=kubernetes-etcd-init-token --initial-advertise-peer-urls "${INITIAL_ADVERTISE_PEER_URLS}"
+       if [[ $? -ne 0 ]]; then
+         echo "Could not restore snapshot from $ETCD_FILE."
+         return 1
+       fi
+
+       # backup etcd host data to /tmp
+       cp -rf {{ .Values.etcd.host_data_path }} /tmp
+
+       # Remove {{ .Values.etcd.host_data_path }}
+       rm -rf {{ .Values.etcd.host_data_path }}
+
+       # Copy snapshot to {{ .Values.etcd.host_data_path }}
+       cp -rf $NAME.etcd/member/ {{ .Values.etcd.host_data_path }}
+       if [[ $? -ne 0 ]]; then
+         echo "Could not copy snapshot to $NAME."
+         return 1
+       fi
+
+       # Delete etcd anchor pod
+       kubectl delete pods -n $POD_NAMESPACE $ANCHOR_POD
+       if [[ $? -ne 0 ]]; then
+         echo "Could not delete $ANCHOR_POD pod."
+         return 1
+       fi
+
+       # Check for pod status
+       kubectl wait -n $POD_NAMESPACE --timeout=15m --for condition=ready pods -l 'application={{ .Values.service.name | replace "-etcd" "" }},component in (etcd,etcd-anchor)'
+       if [[ $? -eq 0 ]]; then
+         echo "Database restore Successful."
+       else
+         echo "Database restore Failed."
+         return 1
+       fi
+
+  else
+    echo "No database file available to restore from."
+    return 1
+  fi
+  return 0
+}
+
+# Call the CLI interpreter, providing the archive directory path and the
+# user arguments passed in
+cli_main ${ARGS[@]}
--- a/charts/etcd/templates/configmap-bin.yaml
+++ b/charts/etcd/templates/configmap-bin.yaml
@ -29,6 +29,16 @@ data:
 {{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
  readiness: |+
 {{ tuple "bin/_readiness.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  etcdbackup: |+
+  backup_etcd.sh: |+
 {{ tuple "bin/_etcdbackup.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  restore_etcd.sh: |+
+{{ tuple "bin/_etcdrestore.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  backup_main.sh: |+
+{{- include "helm-toolkit.scripts.db-backup-restore.backup_main" . | indent 4 }}
+  restore_main.sh: |+
+{{- include "helm-toolkit.scripts.db-backup-restore.restore_main" . | indent 4 }}
+{{- if .Values.manifests.job_ks_user }}
+  ks-user.sh: |
+{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }}
+{{- end }}
 {{- end }}
--- a/charts/etcd/templates/cron-job-etcd-backup.yaml
+++ b/charts/etcd/templates/cron-job-etcd-backup.yaml
@ -30,6 +30,16 @@ rules:
    verbs:
      - get
      - list
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/exec
+    verbs:
+      - create
+      - delete
+      - get
+      - list
 ---
 apiVersion: v1
 kind: ServiceAccount
@ -76,7 +86,7 @@ spec:
            {{ .Values.labels.anchor.node_selector_key }}: {{ .Values.labels.anchor.node_selector_value }}
          containers:
            - name: etcd-backup
-              image: {{ .Values.images.tags.etcdctl }}
+              image: {{ .Values.images.tags.etcdctl_backup }}
              imagePullPolicy: {{ .Values.images.pull_policy }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.etcd_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
              env:
@ -96,8 +106,23 @@ spec:
                  value: https://$(POD_IP):{{ .Values.network.service_client.target_port }}
                - name: PEER_ENDPOINT
                  value: https://$(POD_IP):{{ .Values.network.service_peer.target_port }}
+                - name: POD_NAMESPACE
+                  value: {{ $envAll.Release.Namespace }}
+                - name: REMOTE_BACKUP_ENABLED
+                  value: "{{ .Values.backup.remote_backup.enabled }}"
+{{- if .Values.backup.remote_backup.enabled }}
+                - name: REMOTE_BACKUP_DAYS_TO_KEEP
+                  value: "{{ .Values.backup.remote_backup.days_to_keep }}"
+                - name: CONTAINER_NAME
+                  value: "{{ .Values.backup.remote_backup.container_name }}"
+                - name: STORAGE_POLICY
+                  value: "{{ .Values.backup.remote_backup.storage_policy }}"
+{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.kubernetes }}
+{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }}
+{{- end }}
+{{- end }}
              command:
-                - /tmp/bin/etcdbackup
+                - /tmp/bin/backup_etcd.sh
              volumeMounts:
                - name: {{ .Values.service.name }}-bin
                  mountPath: /tmp/bin
@ -106,7 +131,7 @@ spec:
                - name: {{ .Values.service.name }}-keys
                  mountPath: /etc/etcd/tls/keys
                - name: etcd-backup
-                  mountPath: /var/lib/etcd/backup
+                  mountPath: {{ .Values.backup.host_backup_path }}
          volumes:
          - name: {{ .Values.service.name }}-bin
            configMap:
--- a/charts/etcd/templates/job-ks-user.yaml
+++ b/charts/etcd/templates/job-ks-user.yaml
@ -0,0 +1,23 @@
+{{/*
+Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_ks_user }}
+{{- $serviceName := .Values.service.name | replace "-etcd" "" }}
+{{ $cm_name := printf "%s-bin" .Values.service.name }}
+{{- $_ := set .Values.endpoints.identity.auth $serviceName .Values.endpoints.identity.auth.kubernetes }}
+{{- $ksUserJob := dict "envAll" . "serviceName" $serviceName "configMapBin" $cm_name "serviceUser" $serviceName -}}
+{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }}
+{{- end }}
--- a/charts/etcd/templates/secret-backup-restore.yaml
+++ b/charts/etcd/templates/secret-backup-restore.yaml
@ -0,0 +1,26 @@
+{{/*
+This manifest results a secret being created which has the key information
+needed for backing up and restoring the etcd database.
+*/}}
+
+{{- if and .Values.backup.enabled .Values.manifests.secret_backup_restore }}
+
+{{- $envAll := . }}
+{{- $userClass := "backup_restore" }}
+{{- $secretName := index $envAll.Values.secrets.etcd $userClass }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+  BACKUP_ENABLED: {{ $envAll.Values.backup.enabled | quote | b64enc }}
+  BACKUP_BASE_PATH: {{ $envAll.Values.backup.host_backup_path | b64enc }}
+  LOCAL_DAYS_TO_KEEP: {{ $envAll.Values.backup.no_backup_keep | quote | b64enc }}
+  REMOTE_BACKUP_ENABLED: {{ $envAll.Values.backup.remote_backup.enabled | quote | b64enc }}
+  REMOTE_BACKUP_CONTAINER: {{ $envAll.Values.backup.remote_backup.container_name | b64enc }}
+  REMOTE_BACKUP_DAYS_TO_KEEP: {{ $envAll.Values.backup.remote_backup.days_to_keep | quote | b64enc }}
+  REMOTE_BACKUP_STORAGE_POLICY: {{ $envAll.Values.backup.remote_backup.storage_policy | b64enc }}
+...
+{{- end }}
--- a/charts/etcd/templates/secret-rgw.yaml
+++ b/charts/etcd/templates/secret-rgw.yaml
@ -0,0 +1,66 @@
+{{/*
+This manifest results in two secrets being created:
+  1) Keystone "etcd" secret, which is needed to access the cluster
+     (remote or same cluster) for storing etcd backups. If the
+     cluster is remote, the auth_url would be non-null.
+  2) Keystone "admin" secret, which is needed to create the "etcd"
+     keystone account mentioned above. This may not be needed if the
+     account is in a remote cluster (auth_url is non-null in that case).
+*/}}
+
+{{- if .Values.backup.remote_backup.enabled }}
+
+{{- $envAll := . }}
+{{- $userClass := .Values.service.name | replace "-etcd" "" }}
+{{- $serviceName := $envAll.Values.service.name }}
+{{- $secretName := printf "%s" (index $envAll.Values.secrets.identity $userClass) }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+{{- $identityClass := index .Values.endpoints.identity.auth $userClass }}
+{{- if $identityClass.auth_url }}
+  OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }}
+{{- else }}
+  OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
+{{- end }}
+  OS_REGION_NAME: {{ $identityClass.region_name | b64enc }}
+  OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }}
+  OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }}
+  OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }}
+  OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }}
+  OS_USERNAME: {{ $identityClass.username | b64enc }}
+  OS_PASSWORD: {{ $identityClass.password | b64enc }}
+  OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
+...
+{{- if .Values.manifests.job_ks_user }}
+{{- $userClass := "admin" }}
+{{- $serviceName := $envAll.Values.service.name }}
+{{- $secretName := printf "%s" (index $envAll.Values.secrets.identity $userClass) }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+{{- $identityClass := index .Values.endpoints.identity.auth $userClass }}
+{{- if $identityClass.auth_url }}
+  OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }}
+{{- else }}
+  OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
+{{- end }}
+  OS_REGION_NAME: {{ $identityClass.region_name | b64enc }}
+  OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }}
+  OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }}
+  OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }}
+  OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }}
+  OS_USERNAME: {{ $identityClass.username | b64enc }}
+  OS_PASSWORD: {{ $identityClass.password | b64enc }}
+  OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
+...
+{{- end }}
+{{- end }}
--- a/charts/etcd/values.yaml
+++ b/charts/etcd/values.yaml
@ -16,12 +16,23 @@ images:
  tags:
    etcd: quay.io/coreos/etcd:v3.4.2
    etcdctl: quay.io/coreos/etcd:v3.4.2
+    etcdctl_backup: "quay.io/airshipit/porthole-etcdctl-utility:latest-ubuntu_bionic"
+    dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
+    ks_user: docker.io/openstackhelm/heat:stein-ubuntu_bionic
  pull_policy: "IfNotPresent"
+  local_registry:
+    active: false
+    exclude:
+      - dep_check
+      - image_repo_sync

 labels:
  anchor:
    node_selector_key: etcd-example
    node_selector_value: enabled
+  job:
+    node_selector_key: example-etcd
+    node_selector_value: enabled

 anchor:
  dns_policy: ClusterFirstWithHostNet
@ -49,11 +60,56 @@ etcd:
      - etcdserver=DEBUG
      - security=DEBUG
 backup:
-  host_backup_path: /var/backups/etcd
+  enabled: true
+  host_backup_path: /var/backups
  backup_log_file: /var/log/etcd-backup.log
  no_backup_keep: 10
  etcdctl_dial_timeout: 15s
+  remote_backup:
+    enabled: false
+    container_name: etcd
+    days_to_keep: 14
+    storage_policy: default-placement

+endpoints:
+  identity:
+    name: backup-storage-auth
+    namespace: null
+    auth:
+      example-admin:
+        # Auth URL of null indicates local authentication
+        # HTK will form the URL unless specified here
+        auth_url: null
+        region_name: RegionOne
+        username: example-admin
+        password: password
+        project_name: admin
+        user_domain_name: default
+        project_domain_name: default
+      example-etcd:
+        # Auth URL of null indicates local authentication
+        # HTK will form the URL unless specified here
+        auth_url: null
+        role: admin
+        region_name: RegionOne
+        username: example-etcd-backup-user
+        password: password
+        project_name: service
+        user_domain_name: service
+        project_domain_name: service
+    hosts:
+      default: keystone
+      internal: keystone-api
+    host_fqdn_override:
+      default: null
+    path:
+      default: /v3
+    scheme:
+      default: 'http'
+    port:
+      api:
+        default: 80
+        internal: 5000
 network:
  service_client:
    name: service_client
@ -88,6 +144,11 @@ secrets:
    tls:
      cert: placeholder
      key: placeholder
+  etcd:
+    backup_restore: etcd-backup-restore
+  identity:
+    example-admin: example-admin-user
+    example-etcd: example-backup-user

 nodes:
  - name: example-0
@ -101,9 +162,9 @@ nodes:

 dependencies:
   static:
-     etcd_backup:
+     backup_etcd:
       jobs:
-         - etcd_backup_job
+         - etcd-ks-user
 pod:
  security_context:
    anchor:
@ -113,6 +174,17 @@ pod:
        etcdctl:
          runAsUser: 0
          readOnlyRootFilesystem: false
+    etcd_backup:
+      pod:
+        runAsUser: 65534
+      container:
+        backup_perms:
+          runAsUser: 0
+          readOnlyRootFilesystem: true
+        etcd_backup:
+          runAsUser: 65534
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
    etcd:
      pod:
        runAsUser: 65534
@ -183,6 +255,13 @@ pod:
        requests:
          memory: "128Mi"
          cpu: "100m"
+      ks_user:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
  mandatory_access_control:
    type: apparmor
    # requires override for a specific use case e.g. calico-etcd or kubernetes-etcd
@ -215,6 +294,8 @@ manifests:
  configmap_etc: true
  daemonset_anchor: true
  secret: true
+  secret_backup_restore: false
  service: true
  test_etcd_health: true
  cron_etcd_backup: true
+  job_ks_user: false
--- a/examples/basic/armada-resources.yaml
+++ b/examples/basic/armada-resources.yaml
@ -112,7 +112,7 @@ data:
    type: git
    location: https://git.openstack.org/openstack/openstack-helm-infra
    subpath: helm-toolkit
-    reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
+    reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
  dependencies: []
 ---
 schema: armada/Chart/v1
@ -136,7 +136,7 @@ data:
    type: git
    location: https://git.openstack.org/openstack/openstack-helm-infra
    subpath: helm-toolkit
-    reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
+    reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
  dependencies: []
 ---
 schema: armada/Chart/v1
--- a/examples/complete/armada-resources.yaml
+++ b/examples/complete/armada-resources.yaml
@ -153,7 +153,7 @@ data:
    type: git
    location: https://git.openstack.org/openstack/openstack-helm-infra
    subpath: helm-toolkit
-    reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
+    reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
  dependencies: []
 ---
 schema: armada/Chart/v1
@ -178,7 +178,7 @@ data:
    type: git
    location: https://git.openstack.org/openstack/openstack-helm-infra
    subpath: helm-toolkit
-    reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
+    reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
  dependencies: []
 ---
 schema: armada/Chart/v1
--- a/examples/containerd/armada-resources.yaml
+++ b/examples/containerd/armada-resources.yaml
@ -112,7 +112,7 @@ data:
    type: git
    location: https://opendev.org/openstack/openstack-helm-infra.git
    subpath: helm-toolkit
-    reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
+    reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
  dependencies: []
 ---
 schema: armada/Chart/v1
@ -136,7 +136,7 @@ data:
    type: git
    location: https://opendev.org/openstack/openstack-helm-infra.git
    subpath: helm-toolkit
-    reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
+    reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
  dependencies: []
 ---
 schema: armada/Chart/v1
--- a/examples/gate/armada-resources.yaml
+++ b/examples/gate/armada-resources.yaml
@ -112,7 +112,7 @@ data:
    type: git
    location: https://opendev.org/openstack/openstack-helm-infra.git
    subpath: helm-toolkit
-    reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
+    reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
  dependencies: []
 ---
 schema: armada/Chart/v1
@ -136,7 +136,7 @@ data:
    type: git
    location: https://opendev.org/openstack/openstack-helm-infra.git
    subpath: helm-toolkit
-    reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
+    reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
  dependencies: []
 ---
 schema: armada/Chart/v1
--- a/tools/helm_tk.sh
+++ b/tools/helm_tk.sh
@ -20,7 +20,7 @@ HELM=${1}
 HELM_PIDFILE=${2}
 SERVE_DIR=$(mktemp -d)

-HTK_STABLE_COMMIT=${HTK_COMMIT:-"74f3eb5824f7c52173088d63297f36769ed77a4e"}
+HTK_STABLE_COMMIT=${HTK_COMMIT:-"1da7a5b0f8b66f2012e664de4ee7240627385210"}

 ${HELM} init --client-only