summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSreejith Punnapuzha <Sreejith.Punnapuzha@outlook.com>2018-09-06 12:29:16 -0500
committerSreejith Punnapuzha <Sreejith.Punnapuzha@outlook.com>2019-02-25 14:58:46 +0000
commitb65752d7c52ee39b781fa378776e6c54c938933e (patch)
tree8dceff9571c1c6200ce7f376ac4bab44d79446d6
parent07e29087774783740389af31b1108059a6ee9f98 (diff)
Implements: etcd backup
This is an effort to impletment etcd backup. This will create a k8s cron job to take a regular backup. Change-Id: If2c89ac01540c0f13f9b57a6833a8ea770379717 Signed-off-by: Sreejith Punnapuzha <Sreejith.Punnapuzha@outlook.com>
Notes
Notes (review): Code-Review+1: Vladyslav Drok <vdrok@mirantis.com> Code-Review+1: Krishna Venkata <kvenkata986@gmail.com> Code-Review+1: Nishant Kumar <nishant.e.kumar@ericsson.com> Code-Review+1: chinasubbareddy mallavarapu <cr3938@att.com> Code-Review+2: Scott Hussey <sthussey@att.com> Code-Review+2: Sean Eagan <sean.eagan@att.com> Workflow+1: Sean Eagan <sean.eagan@att.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Wed, 06 Mar 2019 21:05:42 +0000 Reviewed-on: https://review.openstack.org/600493 Project: openstack/airship-promenade Branch: refs/heads/master
-rw-r--r--charts/etcd/templates/bin/_etcdbackup.tpl61
-rw-r--r--charts/etcd/templates/configmap-bin.yaml2
-rw-r--r--charts/etcd/templates/cron-job-etcd-backup.yaml124
-rw-r--r--charts/etcd/values.yaml27
4 files changed, 214 insertions, 0 deletions
diff --git a/charts/etcd/templates/bin/_etcdbackup.tpl b/charts/etcd/templates/bin/_etcdbackup.tpl
new file mode 100644
index 0000000..d728ba9
--- /dev/null
+++ b/charts/etcd/templates/bin/_etcdbackup.tpl
@@ -0,0 +1,61 @@
1#!/bin/sh
2{{/*
3Copyright 2017 AT&T Intellectual Property. All other rights reserved.
4
5Licensed under the Apache License, Version 2.0 (the "License");
6you may not use this file except in compliance with the License.
7You may obtain a copy of the License at
8
9 http://www.apache.org/licenses/LICENSE-2.0
10
11Unless required by applicable law or agreed to in writing, software
12distributed under the License is distributed on an "AS IS" BASIS,
13WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14See the License for the specific language governing permissions and
15limitations under the License.
16*/}}
17set -ex
18BACKUP_DIR="/var/lib/etcd/backup"
19BACKUP_LOG={{ .Values.backup.backup_log_file | quote }}
20NUM_TO_KEEP={{ .Values.backup.no_backup_keep | quote }}
21SKIP_BACKUP=0
22
23etcdbackup() {
24 etcdctl snapshot save $BACKUP_DIR/etcd-backup-$(date +"%m-%d-%Y-%H-%M-%S").db >> $BACKUP_LOG
25 BACKUP_RETURN_CODE=$?
26 if [[ $BACKUP_RETURN_CODE != 0 ]]; then
27 echo "There was an error backing up the databases. Return code was $BACKUP_RETURN_CODE."
28 exit $BACKUP_RETURN_CODE
29 fi
30 LATEST_BACKUP=`ls -t $BACKUP_DIR | head -1`
31 echo "Archiving $LATEST_BACKUP..."
32 cd $BACKUP_DIR
33 tar -czf $BACKUP_DIR/$LATEST_BACKUP.tar.gz $LATEST_BACKUP
34 rm -rf $LATEST_BACKUP
35 echo "Clearing earliest backups..."
36 NUM_LOCAL_BACKUPS=`ls -ld $BACKUP_DIR | wc -l`
37 while [ $NUM_LOCAL_BACKUPS -gt $NUM_TO_KEEP ]
38 do
39 EARLIEST_BACKUP=`ls -tr $BACKUP_DIR | head -1`
40 echo "Deleting $EARLIEST_BACKUP..."
41 rm -rf "$BACKUP_DIR/$EARLIEST_BACKUP"
42 NUM_LOCAL_BACKUPS=`ls -ld $BACKUP_DIR | wc -l`
43 done
44}
45
46if ! [ -x "$(which etcdctl)" ]; then
47 echo "ERROR: etcdctl not available, Please use the correct image."
48 SKIP_BACKUP=1
49fi
50
51if [ ! -d "$BACKUP_DIR" ]; then
52 echo "ERROR: $BACKUP_DIR doesn't exist, Backup will not continue"
53 SKIP_BACKUP=1
54fi
55
56if [ $SKIP_BACKUP == '0' ]; then
57 etcdbackup
58else
59 echo "Error: etcd backup failed."
60 exit 1
61fi
diff --git a/charts/etcd/templates/configmap-bin.yaml b/charts/etcd/templates/configmap-bin.yaml
index 1412aeb..7c40911 100644
--- a/charts/etcd/templates/configmap-bin.yaml
+++ b/charts/etcd/templates/configmap-bin.yaml
@@ -29,4 +29,6 @@ data:
29{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} 29{{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
30 readiness: |+ 30 readiness: |+
31{{ tuple "bin/_readiness.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} 31{{ tuple "bin/_readiness.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
32 etcdbackup: |+
33{{ tuple "bin/_etcdbackup.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
32{{- end }} 34{{- end }}
diff --git a/charts/etcd/templates/cron-job-etcd-backup.yaml b/charts/etcd/templates/cron-job-etcd-backup.yaml
new file mode 100644
index 0000000..83f420e
--- /dev/null
+++ b/charts/etcd/templates/cron-job-etcd-backup.yaml
@@ -0,0 +1,124 @@
1{{/*
2Copyright 2017 AT&T Intellectual Property. All other rights reserved.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/}}
16{{- if .Values.manifests.cron_etcd_backup }}
17{{- $envAll := . }}
18{{- $serviceAccountName := "etcd-backup" }}
19{{- $applicationName := "etcd-backup" }}
20---
21apiVersion: rbac.authorization.k8s.io/v1beta1
22kind: Role
23metadata:
24 name: {{ $serviceAccountName }}
25rules:
26 - apiGroups:
27 - ""
28 resources:
29 - secrets
30 verbs:
31 - get
32 - list
33---
34apiVersion: v1
35kind: ServiceAccount
36metadata:
37 labels:
38 component: etcd-backup
39 name: {{ $serviceAccountName }}
40 namespace: {{ $envAll.Release.Namespace }}
41---
42apiVersion: rbac.authorization.k8s.io/v1beta1
43kind: RoleBinding
44metadata:
45 name: {{ $serviceAccountName }}
46roleRef:
47 apiGroup: rbac.authorization.k8s.io
48 kind: Role
49 name: {{ $serviceAccountName }}
50subjects:
51 - kind: ServiceAccount
52 name: {{ $serviceAccountName }}
53 namespace: {{ $envAll.Release.Namespace }}
54---
55apiVersion: batch/v1beta1
56kind: CronJob
57metadata:
58 name: etcd-backup
59spec:
60 schedule: {{ .Values.jobs.etcd_backup.cron | quote }}
61 successfulJobsHistoryLimit: {{ .Values.jobs.etcd_backup.history.success }}
62 failedJobsHistoryLimit: {{ .Values.jobs.etcd_backup.history.failed }}
63 concurrencyPolicy: Forbid
64 jobTemplate:
65 metadata:
66 labels:
67{{ tuple $envAll $applicationName "etcd-anchor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
68 spec:
69 template:
70 spec:
71 serviceAccountName: {{ $serviceAccountName }}
72 restartPolicy: OnFailure
73 nodeSelector:
74 {{ .Values.labels.anchor.node_selector_key }}: {{ .Values.labels.anchor.node_selector_value }}
75 containers:
76 - name: etcd-backup
77 image: {{ .Values.images.tags.etcdctl }}
78 imagePullPolicy: {{ .Values.images.pull_policy }}
79{{ tuple $envAll $envAll.Values.pod.resources.jobs.etcd_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
80 env:
81 - name: ETCDCTL_API
82 value: '3'
83 - name: ETCDCTL_DIAL_TIMEOUT
84 value: {{ .Values.backup.etcdctl_dial_timeout }}
85 - name: ETCDCTL_ENDPOINTS
86 value: https://{{ .Values.anchor.etcdctl_endpoint }}:{{ .Values.network.service_client.port }}
87 - name: ETCDCTL_CACERT
88 value: /etc/etcd/tls/certs/client-ca.pem
89 - name: ETCDCTL_CERT
90 value: /etc/etcd/tls/certs/anchor-etcd-client.pem
91 - name: ETCDCTL_KEY
92 value: /etc/etcd/tls/keys/anchor-etcd-client-key.pem
93 - name: CLIENT_ENDPOINT
94 value: https://$(POD_IP):{{ .Values.network.service_client.target_port }}
95 - name: PEER_ENDPOINT
96 value: https://$(POD_IP):{{ .Values.network.service_peer.target_port }}
97 command:
98 - /tmp/bin/etcdbackup
99 volumeMounts:
100 - name: {{ .Values.service.name }}-bin
101 mountPath: /tmp/bin
102 - name: {{ .Values.service.name }}-certs
103 mountPath: /etc/etcd/tls/certs
104 - name: {{ .Values.service.name }}-keys
105 mountPath: /etc/etcd/tls/keys
106 - name: etcd-backup
107 mountPath: /var/lib/etcd/backup
108 volumes:
109 - name: {{ .Values.service.name }}-bin
110 configMap:
111 name: {{ .Values.service.name }}-bin
112 defaultMode: 0555
113 - name: {{ .Values.service.name }}-certs
114 configMap:
115 name: {{ .Values.service.name }}-certs
116 defaultMode: 0444
117 - name: etcd-backup
118 hostPath:
119 path: {{ .Values.backup.host_backup_path }}
120 - name: {{ .Values.service.name }}-keys
121 secret:
122 secretName: {{ .Values.service.name }}-keys
123 defaultMode: 0444
124{{- end }}
diff --git a/charts/etcd/values.yaml b/charts/etcd/values.yaml
index 15ef3bf..a115672 100644
--- a/charts/etcd/values.yaml
+++ b/charts/etcd/values.yaml
@@ -39,6 +39,12 @@ etcd:
39 host_data_path: /var/lib/etcd/example 39 host_data_path: /var/lib/etcd/example
40 cleanup_data: true 40 cleanup_data: true
41 41
42backup:
43 host_backup_path: /var/lib/etcd/backup
44 backup_log_file: /var/log/etcd-backup.log
45 no_backup_keep: 10
46 etcdctl_dial_timeout: 15s
47
42network: 48network:
43 service_client: 49 service_client:
44 name: service_client 50 name: service_client
@@ -83,6 +89,11 @@ nodes:
83 cert: placeholder 89 cert: placeholder
84 key: placeholder 90 key: placeholder
85 91
92dependencies:
93 static:
94 etcd_backup:
95 jobs:
96 - etcd_backup_job
86pod: 97pod:
87 mounts: 98 mounts:
88 daemonset_anchor: 99 daemonset_anchor:
@@ -123,6 +134,21 @@ pod:
123 requests: 134 requests:
124 memory: "128Mi" 135 memory: "128Mi"
125 cpu: "100m" 136 cpu: "100m"
137 jobs:
138 etcdbackup:
139 limits:
140 memory: "128Mi"
141 cpu: "100m"
142 requests:
143 memory: "128Mi"
144 cpu: "100m"
145
146jobs:
147 etcd_backup:
148 cron: "0 */12 * * *"
149 history:
150 success: 3
151 failed: 1
126 152
127manifests: 153manifests:
128 configmap_bin: true 154 configmap_bin: true
@@ -132,3 +158,4 @@ manifests:
132 secret: true 158 secret: true
133 service: true 159 service: true
134 test_etcd_health: true 160 test_etcd_health: true
161 cron_etcd_backup: true