Merge "ETCD remote backup enhancements"

This commit is contained in:
Zuul 2020-06-30 22:23:33 +00:00 committed by Gerrit Code Review
commit c6c7a3accd
13 changed files with 384 additions and 40 deletions

View File

@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
{{/*
Copyright 2017 AT&T Intellectual Property. All other rights reserved.
@ -14,49 +14,51 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
BACKUP_DIR="/var/lib/etcd/backup"
set -x
BACKUP_DIR={{ .Values.backup.host_backup_path }}
BACKUP_LOG={{ .Values.backup.backup_log_file | quote }}
NUM_TO_KEEP={{ .Values.backup.no_backup_keep | quote }}
REMOTE_BACKUP_DAYS_TO_KEEP={{ .Values.backup.remote_backup.days_to_keep | quote }}
BACKUP_FILE_NAME={{ .Values.service.name | quote }}
SKIP_BACKUP=0
etcdbackup() {
etcdctl snapshot save $BACKUP_DIR/$BACKUP_FILE_NAME-backup-$(date +"%m-%d-%Y-%H-%M-%S").db >> $BACKUP_LOG
source /tmp/bin/backup_main.sh
# Export the variables required by the framework
# Note: REMOTE_BACKUP_ENABLED and CONTAINER_NAME are already exported
export DB_NAMESPACE=${POD_NAMESPACE}
export DB_NAME="etcd"
export LOCAL_DAYS_TO_KEEP=$NUM_TO_KEEP
export REMOTE_DAYS_TO_KEEP=$REMOTE_BACKUP_DAYS_TO_KEEP
export ARCHIVE_DIR=${BACKUP_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive
dump_databases_to_directory() {
TMP_DIR=$1
LOG_FILE=${2:-BACKUP_LOG}
cd $TMP_DIR
etcdctl snapshot save --command-timeout=5m $TMP_DIR/$BACKUP_FILE_NAME.$DB_NAMESPACE.all.db >> $LOG_FILE
BACKUP_RETURN_CODE=$?
if [[ $BACKUP_RETURN_CODE != 0 ]]; then
echo "There was an error backing up the databases. Return code was $BACKUP_RETURN_CODE."
log ERROR $DB_NAME "There was an error backing up the databases." $LOG_FILE
exit $BACKUP_RETURN_CODE
fi
LATEST_BACKUP=`ls -t1 $BACKUP_DIR | grep $BACKUP_FILE_NAME | head -1`
echo "Archiving $LATEST_BACKUP..."
cd $BACKUP_DIR
tar -czf $BACKUP_DIR/$LATEST_BACKUP.tar.gz $LATEST_BACKUP
rm -rf $LATEST_BACKUP
echo "Clearing earliest backups..."
NUM_LOCAL_BACKUPS=`ls -1 $BACKUP_DIR | grep $BACKUP_FILE_NAME | wc -l`
while [ $NUM_LOCAL_BACKUPS -gt $NUM_TO_KEEP ]
do
EARLIEST_BACKUP=`ls -tr1 $BACKUP_DIR | grep $BACKUP_FILE_NAME | head -1`
echo "Deleting $EARLIEST_BACKUP..."
rm -rf "$BACKUP_DIR/$EARLIEST_BACKUP"
NUM_LOCAL_BACKUPS=`ls -1 $BACKUP_DIR | grep $BACKUP_FILE_NAME | wc -l`
done
}
if ! [ -x "$(which etcdctl)" ]; then
echo "ERROR: etcdctl not available, Please use the correct image."
log ERROR $DB_NAME "etcdctl not available, Please use the correct image." $LOG_FILE
SKIP_BACKUP=1
fi
if [ ! -d "$BACKUP_DIR" ]; then
echo "ERROR: $BACKUP_DIR doesn't exist, Backup will not continue"
log ERROR $DB_NAME "$BACKUP_DIR doesn't exist, Backup will not continue" $LOG_FILE
SKIP_BACKUP=1
fi
if [ $SKIP_BACKUP -eq 0 ]; then
etcdbackup
# Call main program to start the database backup
backup_databases
else
echo "Error: etcd backup failed."
log ERROR $DB_NAME "etcd backup failed." $LOG_FILE
exit 1
fi

View File

@ -0,0 +1,111 @@
#!/bin/bash
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -x
# Capture the user's command line arguments
ARGS=("$@")
source /tmp/restore_main.sh
# Export the variables needed by the framework
export DB_NAME="etcd"
export DB_NAMESPACE=${POD_NAMESPACE}
export SINGLE_DB_NAME_DIR=${ETCD_BACKUP_BASE_PATH}/db/${DB_NAMESPACE}/${DB_NAME}/archive
# Extract all databases from an archive and put them in the requested
# file.
get_databases() {
TMP_DIR=$1
DB_FILE=$2
ETCD_FILE={{ .Values.service.name }}.$POD_NAMESPACE.all.db
if [[ -e $TMP_DIR/$ETCD_FILE ]]; then
grep 'CREATE DATABASE' $TMP_DIR/$ETCD_FILE | awk '{ print $3 }' > $DB_FILE
else
# no databases - just touch the file
touch $DB_FILE
fi
}
restore_single_db() {
SINGLE_DB_NAME=$1
TMP_DIR=$2
ANCHOR_POD=$SINGLE_DB_NAME
if [[ -f $TMP_DIR/$ETCD_FILE ]]; then
# Check etcd-anchor pod
if [[ ! $(kubectl get pods -n $POD_NAMESPACE $ANCHOR_POD) ]]; then
echo "Could not find pod $ANCHOR_POD."
return 1
fi
# Copy backup to etcd-anchor
kubectl cp -n $POD_NAMESPACE $TMP_DIR/$ETCD_FILE $ANCHOR_POD:/
if [[ $? -ne 0 ]]; then
echo "Could not copy backup to $ANCHOR_POD."
return 1
fi
# Node Name
NAME=$(kubectl get pods -n $POD_NAMESPACE $ANCHOR_POD -o jsonpath={.spec.nodeName})
# Initial Cluster
INITIAL_CLUSTER="$(etcdctl member list|awk -F , '{gsub (" ", "", $0);printf "%s=%s,", $3,$4}')"
INITIAL_ADVERTISE_PEER_URLS=$(kubectl exec -it -n $POD_NAMESPACE $ANCHOR_POD -- env| grep PEER |awk -F = '{print $2}')
# Restore snapshot
kubectl exec -it -n $POD_NAMESPACE $ANCHOR_POD -- env ETCD_FILE=$ETCD_FILE NAME=$NAME INITIAL_CLUSTER=$INITIAL_CLUSTER INITIAL_ADVERTISE_PEER_URLS=$INITIAL_ADVERTISE_PEER_URLS;/usr/local/bin/etcdctl snapshot restore $ETCD_FILE --name $NAME --initial-cluster "$INITIAL_CLUSTER" --initial-cluster-token=kubernetes-etcd-init-token --initial-advertise-peer-urls "${INITIAL_ADVERTISE_PEER_URLS}"
if [[ $? -ne 0 ]]; then
echo "Could not restore snapshot from $ETCD_FILE."
return 1
fi
# backup etcd host data to /tmp
cp -rf {{ .Values.etcd.host_data_path }} /tmp
# Remove {{ .Values.etcd.host_data_path }}
rm -rf {{ .Values.etcd.host_data_path }}
# Copy snapshot to {{ .Values.etcd.host_data_path }}
cp -rf $NAME.etcd/member/ {{ .Values.etcd.host_data_path }}
if [[ $? -ne 0 ]]; then
echo "Could not copy snapshot to $NAME."
return 1
fi
# Delete etcd anchor pod
kubectl delete pods -n $POD_NAMESPACE $ANCHOR_POD
if [[ $? -ne 0 ]]; then
echo "Could not delete $ANCHOR_POD pod."
return 1
fi
# Check for pod status
kubectl wait -n $POD_NAMESPACE --timeout=15m --for condition=ready pods -l 'application={{ .Values.service.name | replace "-etcd" "" }},component in (etcd,etcd-anchor)'
if [[ $? -eq 0 ]]; then
echo "Database restore Successful."
else
echo "Database restore Failed."
return 1
fi
else
echo "No database file available to restore from."
return 1
fi
return 0
}
# Call the CLI interpreter, providing the archive directory path and the
# user arguments passed in
cli_main ${ARGS[@]}

View File

@ -29,6 +29,16 @@ data:
{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
readiness: |+
{{ tuple "bin/_readiness.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
etcdbackup: |+
backup_etcd.sh: |+
{{ tuple "bin/_etcdbackup.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
restore_etcd.sh: |+
{{ tuple "bin/_etcdrestore.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
backup_main.sh: |+
{{- include "helm-toolkit.scripts.db-backup-restore.backup_main" . | indent 4 }}
restore_main.sh: |+
{{- include "helm-toolkit.scripts.db-backup-restore.restore_main" . | indent 4 }}
{{- if .Values.manifests.job_ks_user }}
ks-user.sh: |
{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }}
{{- end }}
{{- end }}

View File

@ -30,6 +30,16 @@ rules:
verbs:
- get
- list
- apiGroups:
- ""
resources:
- pods
- pods/exec
verbs:
- create
- delete
- get
- list
---
apiVersion: v1
kind: ServiceAccount
@ -76,7 +86,7 @@ spec:
{{ .Values.labels.anchor.node_selector_key }}: {{ .Values.labels.anchor.node_selector_value }}
containers:
- name: etcd-backup
image: {{ .Values.images.tags.etcdctl }}
image: {{ .Values.images.tags.etcdctl_backup }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.etcd_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
env:
@ -96,8 +106,23 @@ spec:
value: https://$(POD_IP):{{ .Values.network.service_client.target_port }}
- name: PEER_ENDPOINT
value: https://$(POD_IP):{{ .Values.network.service_peer.target_port }}
- name: POD_NAMESPACE
value: {{ $envAll.Release.Namespace }}
- name: REMOTE_BACKUP_ENABLED
value: "{{ .Values.backup.remote_backup.enabled }}"
{{- if .Values.backup.remote_backup.enabled }}
- name: REMOTE_BACKUP_DAYS_TO_KEEP
value: "{{ .Values.backup.remote_backup.days_to_keep }}"
- name: CONTAINER_NAME
value: "{{ .Values.backup.remote_backup.container_name }}"
- name: STORAGE_POLICY
value: "{{ .Values.backup.remote_backup.storage_policy }}"
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.kubernetes }}
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }}
{{- end }}
{{- end }}
command:
- /tmp/bin/etcdbackup
- /tmp/bin/backup_etcd.sh
volumeMounts:
- name: {{ .Values.service.name }}-bin
mountPath: /tmp/bin
@ -106,7 +131,7 @@ spec:
- name: {{ .Values.service.name }}-keys
mountPath: /etc/etcd/tls/keys
- name: etcd-backup
mountPath: /var/lib/etcd/backup
mountPath: {{ .Values.backup.host_backup_path }}
volumes:
- name: {{ .Values.service.name }}-bin
configMap:

View File

@ -0,0 +1,23 @@
{{/*
Copyright 2020 AT&T Intellectual Property. All other rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.job_ks_user }}
{{- $serviceName := .Values.service.name | replace "-etcd" "" }}
{{ $cm_name := printf "%s-bin" .Values.service.name }}
{{- $_ := set .Values.endpoints.identity.auth $serviceName .Values.endpoints.identity.auth.kubernetes }}
{{- $ksUserJob := dict "envAll" . "serviceName" $serviceName "configMapBin" $cm_name "serviceUser" $serviceName -}}
{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }}
{{- end }}

View File

@ -0,0 +1,26 @@
{{/*
This manifest results a secret being created which has the key information
needed for backing up and restoring the etcd database.
*/}}
{{- if and .Values.backup.enabled .Values.manifests.secret_backup_restore }}
{{- $envAll := . }}
{{- $userClass := "backup_restore" }}
{{- $secretName := index $envAll.Values.secrets.etcd $userClass }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
type: Opaque
data:
BACKUP_ENABLED: {{ $envAll.Values.backup.enabled | quote | b64enc }}
BACKUP_BASE_PATH: {{ $envAll.Values.backup.host_backup_path | b64enc }}
LOCAL_DAYS_TO_KEEP: {{ $envAll.Values.backup.no_backup_keep | quote | b64enc }}
REMOTE_BACKUP_ENABLED: {{ $envAll.Values.backup.remote_backup.enabled | quote | b64enc }}
REMOTE_BACKUP_CONTAINER: {{ $envAll.Values.backup.remote_backup.container_name | b64enc }}
REMOTE_BACKUP_DAYS_TO_KEEP: {{ $envAll.Values.backup.remote_backup.days_to_keep | quote | b64enc }}
REMOTE_BACKUP_STORAGE_POLICY: {{ $envAll.Values.backup.remote_backup.storage_policy | b64enc }}
...
{{- end }}

View File

@ -0,0 +1,66 @@
{{/*
This manifest results in two secrets being created:
1) Keystone "etcd" secret, which is needed to access the cluster
(remote or same cluster) for storing etcd backups. If the
cluster is remote, the auth_url would be non-null.
2) Keystone "admin" secret, which is needed to create the "etcd"
keystone account mentioned above. This may not be needed if the
account is in a remote cluster (auth_url is non-null in that case).
*/}}
{{- if .Values.backup.remote_backup.enabled }}
{{- $envAll := . }}
{{- $userClass := .Values.service.name | replace "-etcd" "" }}
{{- $serviceName := $envAll.Values.service.name }}
{{- $secretName := printf "%s" (index $envAll.Values.secrets.identity $userClass) }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
type: Opaque
data:
{{- $identityClass := index .Values.endpoints.identity.auth $userClass }}
{{- if $identityClass.auth_url }}
OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }}
{{- else }}
OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
{{- end }}
OS_REGION_NAME: {{ $identityClass.region_name | b64enc }}
OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }}
OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }}
OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }}
OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }}
OS_USERNAME: {{ $identityClass.username | b64enc }}
OS_PASSWORD: {{ $identityClass.password | b64enc }}
OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
...
{{- if .Values.manifests.job_ks_user }}
{{- $userClass := "admin" }}
{{- $serviceName := $envAll.Values.service.name }}
{{- $secretName := printf "%s" (index $envAll.Values.secrets.identity $userClass) }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
type: Opaque
data:
{{- $identityClass := index .Values.endpoints.identity.auth $userClass }}
{{- if $identityClass.auth_url }}
OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }}
{{- else }}
OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }}
{{- end }}
OS_REGION_NAME: {{ $identityClass.region_name | b64enc }}
OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }}
OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }}
OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }}
OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }}
OS_USERNAME: {{ $identityClass.username | b64enc }}
OS_PASSWORD: {{ $identityClass.password | b64enc }}
OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }}
...
{{- end }}
{{- end }}

View File

@ -16,12 +16,23 @@ images:
tags:
etcd: quay.io/coreos/etcd:v3.4.2
etcdctl: quay.io/coreos/etcd:v3.4.2
etcdctl_backup: "quay.io/airshipit/porthole-etcdctl-utility:latest-ubuntu_bionic"
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
ks_user: docker.io/openstackhelm/heat:stein-ubuntu_bionic
pull_policy: "IfNotPresent"
local_registry:
active: false
exclude:
- dep_check
- image_repo_sync
labels:
anchor:
node_selector_key: etcd-example
node_selector_value: enabled
job:
node_selector_key: example-etcd
node_selector_value: enabled
anchor:
dns_policy: ClusterFirstWithHostNet
@ -49,11 +60,56 @@ etcd:
- etcdserver=DEBUG
- security=DEBUG
backup:
host_backup_path: /var/backups/etcd
enabled: true
host_backup_path: /var/backups
backup_log_file: /var/log/etcd-backup.log
no_backup_keep: 10
etcdctl_dial_timeout: 15s
remote_backup:
enabled: false
container_name: etcd
days_to_keep: 14
storage_policy: default-placement
endpoints:
identity:
name: backup-storage-auth
namespace: null
auth:
example-admin:
# Auth URL of null indicates local authentication
# HTK will form the URL unless specified here
auth_url: null
region_name: RegionOne
username: example-admin
password: password
project_name: admin
user_domain_name: default
project_domain_name: default
example-etcd:
# Auth URL of null indicates local authentication
# HTK will form the URL unless specified here
auth_url: null
role: admin
region_name: RegionOne
username: example-etcd-backup-user
password: password
project_name: service
user_domain_name: service
project_domain_name: service
hosts:
default: keystone
internal: keystone-api
host_fqdn_override:
default: null
path:
default: /v3
scheme:
default: 'http'
port:
api:
default: 80
internal: 5000
network:
service_client:
name: service_client
@ -88,6 +144,11 @@ secrets:
tls:
cert: placeholder
key: placeholder
etcd:
backup_restore: etcd-backup-restore
identity:
example-admin: example-admin-user
example-etcd: example-backup-user
nodes:
- name: example-0
@ -101,9 +162,9 @@ nodes:
dependencies:
static:
etcd_backup:
backup_etcd:
jobs:
- etcd_backup_job
- etcd-ks-user
pod:
security_context:
anchor:
@ -113,6 +174,17 @@ pod:
etcdctl:
runAsUser: 0
readOnlyRootFilesystem: false
etcd_backup:
pod:
runAsUser: 65534
container:
backup_perms:
runAsUser: 0
readOnlyRootFilesystem: true
etcd_backup:
runAsUser: 65534
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
etcd:
pod:
runAsUser: 65534
@ -190,6 +262,13 @@ pod:
requests:
memory: "128Mi"
cpu: "100m"
ks_user:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "1024Mi"
cpu: "2000m"
mandatory_access_control:
type: apparmor
# requires override for a specific use case e.g. calico-etcd or kubernetes-etcd
@ -222,6 +301,8 @@ manifests:
configmap_etc: true
daemonset_anchor: true
secret: true
secret_backup_restore: false
service: true
test_etcd_health: true
cron_etcd_backup: true
job_ks_user: false

View File

@ -112,7 +112,7 @@ data:
type: git
location: https://git.openstack.org/openstack/openstack-helm-infra
subpath: helm-toolkit
reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
dependencies: []
---
schema: armada/Chart/v1
@ -136,7 +136,7 @@ data:
type: git
location: https://git.openstack.org/openstack/openstack-helm-infra
subpath: helm-toolkit
reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
dependencies: []
---
schema: armada/Chart/v1

View File

@ -153,7 +153,7 @@ data:
type: git
location: https://git.openstack.org/openstack/openstack-helm-infra
subpath: helm-toolkit
reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
dependencies: []
---
schema: armada/Chart/v1
@ -178,7 +178,7 @@ data:
type: git
location: https://git.openstack.org/openstack/openstack-helm-infra
subpath: helm-toolkit
reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
dependencies: []
---
schema: armada/Chart/v1

View File

@ -112,7 +112,7 @@ data:
type: git
location: https://opendev.org/openstack/openstack-helm-infra.git
subpath: helm-toolkit
reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
dependencies: []
---
schema: armada/Chart/v1
@ -136,7 +136,7 @@ data:
type: git
location: https://opendev.org/openstack/openstack-helm-infra.git
subpath: helm-toolkit
reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
dependencies: []
---
schema: armada/Chart/v1

View File

@ -112,7 +112,7 @@ data:
type: git
location: https://opendev.org/openstack/openstack-helm-infra.git
subpath: helm-toolkit
reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
dependencies: []
---
schema: armada/Chart/v1
@ -136,7 +136,7 @@ data:
type: git
location: https://opendev.org/openstack/openstack-helm-infra.git
subpath: helm-toolkit
reference: b50fae62a4ad0992ce877cd632800e1eed5f71a9
reference: 1da7a5b0f8b66f2012e664de4ee7240627385210
dependencies: []
---
schema: armada/Chart/v1

View File

@ -20,7 +20,7 @@ HELM=${1}
HELM_PIDFILE=${2}
SERVE_DIR=$(mktemp -d)
HTK_STABLE_COMMIT=${HTK_COMMIT:-"74f3eb5824f7c52173088d63297f36769ed77a4e"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"1da7a5b0f8b66f2012e664de4ee7240627385210"}
${HELM} init --client-only