From ad30aa73820edffc8a2b73702df6132a3e6673e5 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Sat, 19 Jan 2019 13:03:59 -0600 Subject: [PATCH] (apiserver) support key rotation - Support key rotation for the etcd encryption key in the apiserver chart - Remove configmap annotations from the apiserver anchor pods as the pod is built to pickup changes in configmap contents without restart. - Also update the apiserver anchor DaemonSet to apps/v1 and make required updates to support that update. Change-Id: I2d18996bbe04bada9da2bce01a502550d3681c97 --- .../apiserver/templates/bin/_key_rotate.tpl | 85 ++++++++++++ charts/apiserver/templates/configmap-bin.yaml | 2 + charts/apiserver/templates/daemonset.yaml | 10 +- .../etc/_kubernetes-apiserver.yaml.tpl | 24 ++++ .../apiserver/templates/job-key-rotate.yaml | 125 ++++++++++++++++++ charts/apiserver/values.yaml | 81 ++++++++---- 6 files changed, 300 insertions(+), 27 deletions(-) create mode 100644 charts/apiserver/templates/bin/_key_rotate.tpl create mode 100644 charts/apiserver/templates/job-key-rotate.yaml diff --git a/charts/apiserver/templates/bin/_key_rotate.tpl b/charts/apiserver/templates/bin/_key_rotate.tpl new file mode 100644 index 00000000..6cdd65d4 --- /dev/null +++ b/charts/apiserver/templates/bin/_key_rotate.tpl @@ -0,0 +1,85 @@ +#!/bin/bash +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -ex + +TEMP_DIR=$(mktemp -d) +ANNOTATION_NAME="airshipit.org/encryption_key" + +get_service_endpoints() { + ns="$1" + svc="$2" + kubectl -n $ns get endpoints -o json $svc | jq '.subsets[0].addresses' | jq '.[] | .targetRef.name' -r + +} + +get_pod_annotation() { + ns="$1" + pod_name="$2" + kubectl -n $ns get pod "$pod_name" -o json | jq ".metadata.annotations.\"${ANNOTATION_NAME}\"" +} + +get_annotations_key() { + echo $ENCRYPTION_ANNOTATION | tr -d ' ' | awk -F':' '{print $1}' +} + +get_encryption_hash() { + echo $ENCRYPTION_ANNOTATION | tr -d ' ' | awk -F':' '{print $2}' +} + +apiserver_compare() { + echo "${apiservers[@]}" | sort | uniq > "${TEMP_DIR}/a.txt" + echo "${updated_apiservers[@]}" | sort | uniq > "${TEMP_DIR}/b.txt" + comm -3 "${TEMP_DIR}/a.txt" "${TEMP_DIR}/b.txt" +} + +{{- $envAll := . }} + + +{{- if and (.Values.conf) (hasKey .Values.conf "encryption_provider") }} + +ENCRYPTION_ANNOTATION='{{ $envAll | include "kubernetes_apiserver.key_annotation" }}' +KUBE_SERVICE_NAMESPACE=${KUBE_SERVICE_NAMESPACE:-"kube-system"} +KUBE_SERVICE_NAME=${KUBE_SERVICE_NAME:-"kubernetes-apiserver"} + +apiservers=( $(get_service_endpoints "$KUBE_SERVICE_NAMESPACE" "$KUBE_SERVICE_NAME")) +updated_apiservers=() + +annotation="$(get_annotations_key)" + +# TODO(sh8121att) add timeout logic +while [[ -n "$(apiserver_compare)" ]]; +do + for pod_name in "${apiservers[@]}"; + do + pod_key=$(get_pod_annotation "$KUBE_SERVICE_NAMESPACE" "$pod_name") + if [ "$pod_key" == "$(get_encryption_hash)" ]; + then + updated_apiservers+=("$pod_name") + fi + done +done + +echo "All apiserver instances have an updated key." + +while true +do + kubectl get secrets --all-namespaces -o json | kubectl replace --validate=false -f - + if [[ $? -eq 0 ]] + then + echo "All secret resources re-encrypted." + exit 0 + fi +done +{{- end -}} diff --git a/charts/apiserver/templates/configmap-bin.yaml b/charts/apiserver/templates/configmap-bin.yaml index 5205587b..34a55f04 100644 --- a/charts/apiserver/templates/configmap-bin.yaml +++ b/charts/apiserver/templates/configmap-bin.yaml @@ -27,4 +27,6 @@ data: {{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} pre_stop: |+ {{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + key-rotate.sh: |+ +{{ tuple "bin/_key_rotate.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/charts/apiserver/templates/daemonset.yaml b/charts/apiserver/templates/daemonset.yaml index cf49cb0e..5620dd34 100644 --- a/charts/apiserver/templates/daemonset.yaml +++ b/charts/apiserver/templates/daemonset.yaml @@ -33,15 +33,17 @@ spec: matchLabels: {{ $labels | indent 6 }} {{ tuple $envAll "kubernetes-apiserver-anchor" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + selector: + matchLabels: +{{ $labels | indent 6 }} template: metadata: labels: {{ $labels | indent 8 }} annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} scheduler.alpha.kubernetes.io/critical-pod: '' configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-certs-hash: {{ tuple "configmap-certs.yaml" . | include "helm-toolkit.utils.hash" }} - configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} spec: nodeSelector: {{ .Values.labels.kubernetes_apiserver.node_selector_key }}: {{ .Values.labels.kubernetes_apiserver.node_selector_value }} @@ -63,8 +65,8 @@ spec: value: /host{{ .Values.anchor.kubelet.manifest_path }}/{{ .Values.service.name }}.yaml - name: PKI_PATH value: /host{{ .Values.apiserver.host_etc_path }}/pki - command: - - /tmp/bin/anchor + command: ["/bin/sh","-c"] + args: ["/tmp/bin/anchor"] lifecycle: preStop: exec: diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl index ee993986..1689dfc0 100644 --- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl +++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl @@ -12,6 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- define "kubernetes_apiserver.key_concat" -}} + {{- range $encProv := .Values.conf.encryption_provider.content.resources -}} + {{- if hasKey (index $encProv "providers" 0) "identity" -}} + {{- printf "%s" (index $encProv "providers" 0 "identity") -}} + {{- else if hasKey (index $encProv "providers" 0) "secretbox" -}} + {{- printf "%s" (index $encProv "providers" 0 "secretbox" "keys" 0 "secret") -}} + {{- else if hasKey (index $encProv "providers" 0) "aescbc" -}} + {{- printf "%s" (index $encProv "providers" 0 "aescbc" "keys" 0 "secret") -}} + {{- else if hasKey (index $encProv "providers" 0) "aesgcm" -}} + {{- printf "%s" (index $encProv "providers" 0 "aesgcm" "keys" 0 "secret") -}} + {{- end -}} + {{- end -}} +{{- end -}} + + +{{- define "kubernetes_apiserver.key_annotation" -}} + {{- if and (.Values.conf) (hasKey .Values.conf "encryption_provider") -}} + {{- $encKey := ( . | include "kubernetes_apiserver.key_concat") -}} +{{ .Values.const.encryption_annotation | quote }}: {{ sha256sum $encKey | quote }} + {{- end -}} +{{- end -}} + + {{- $envAll := . }} --- apiVersion: v1 @@ -23,6 +46,7 @@ metadata: {{ .Values.service.name }}-service: enabled {{ tuple $envAll "kubernetes" "apiserver" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} annotations: + {{ $envAll | include "kubernetes_apiserver.key_annotation" }} {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} spec: hostNetwork: true diff --git a/charts/apiserver/templates/job-key-rotate.yaml b/charts/apiserver/templates/job-key-rotate.yaml new file mode 100644 index 00000000..e748f909 --- /dev/null +++ b/charts/apiserver/templates/job-key-rotate.yaml @@ -0,0 +1,125 @@ +{{/* +Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_key_rotate }} +{{- $envAll := . }} +{{ $serviceAccountName := "apiserver-key-rotate" }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $serviceAccountName }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ $serviceAccountName }}-cluster +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - list + - get + - delete + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }}-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ $serviceAccountName }}-cluster +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ $serviceAccountName }}-{{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - endpoints + - pods + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }}-{{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + namespace: {{ $envAll.Release.Namespace }} + name: {{ $serviceAccountName }}-{{ $envAll.Release.Namespace }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: apiserver-key-rotate + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "kube-apiserver" "key-rotate" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "key_rotate" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: apiserver-key-rotate + image: {{ .Values.images.tags.key_rotate | quote }} + imagePullPolicy: {{ .Values.images.pull_policy | quote }} +{{ tuple $envAll $envAll.Values.pod.resources.key_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: "ANNOTATION_NAME" + value: {{ .Values.const.encryption_annotation | quote }} + command: + - /tmp/key-rotate.sh + volumeMounts: + - name: apiserver-bin + mountPath: /tmp/key-rotate.sh + subPath: key-rotate.sh + readOnly: true + volumes: + - name: apiserver-bin + configMap: + name: {{ .Values.service.name }}-bin + defaultMode: 0555 +... +{{- end }} diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml index c434265c..4278139e 100644 --- a/charts/apiserver/values.yaml +++ b/charts/apiserver/values.yaml @@ -17,6 +17,7 @@ release_group: null # NOTE(mark-burnett): These values are not really configurable -- they live # here to keep the templates cleaner. const: + encryption_annotation: "airshipit.org/encryption_key" command_prefix: - /apiserver - --advertise-address=$(POD_IP) @@ -55,14 +56,24 @@ const: images: tags: + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 anchor: gcr.io/google_containers/hyperkube-amd64:v1.11.6 apiserver: gcr.io/google_containers/hyperkube-amd64:v1.11.6 + key_rotate: gcr.io/google_containers/hyperkube-amd64:v1.11.6 pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync labels: kubernetes_apiserver: node_selector_key: kubernetes-apiserver node_selector_value: enabled + job: + node_selector_key: kubernetes-apiserver + node_selector_value: enabled anchor: dns_policy: Default @@ -85,7 +96,17 @@ conf: # plugins: # - name: EventRateLimit # path: eventconfig.yaml -# +# eventconfig: +# file: eventconfig.yaml +# command_options: [] +# content: +# kind: Configuration +# apiVersion: eventratelimit.admission.k8s.io/v1alpha1 +# limits: +# - type: Server +# qps: 1000 +# burst: 10000 + # Uncomment any of the below to enable enhanced Audit Logging command line options. # # auditpolicy: @@ -98,28 +119,22 @@ conf: # rules: # - level: Metadata # -# eventconfig: -# file: eventconfig.yaml -# command_options: -# - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml' -# content: -# kind: Configuration -# apiVersion: eventratelimit.admission.k8s.io/v1alpha1 -# limits: -# - type: Server -# qps: 1000 -# burst: 10000 -# encryption_provider: -# file: encryption_provider.yaml -# command_option: '' -# content: -# kind: EncryptionConfig -# apiVersion: v1 -# resources: -# - resources: -# - 'secrets' -# providers: -# - identity: {} + encryption_provider: + file: encryption_provider.yaml + command_options: + - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml' + content: + kind: EncryptionConfig + apiVersion: v1 + resources: + - resources: + - 'secrets' + providers: + - secretbox: + keys: + - name: key1 + secret: Xw2UcbjILTJM6QiFZ0WPSbUvjtoT8OJC/Nl8qqYWjGk= + - identity: {} apiserver: arguments: @@ -181,6 +196,18 @@ secrets: cert: null key: null +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - apiserver-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + key_rotate: {} + # typically overriden by environmental # values, but should include all endpoints # required by this chart @@ -236,6 +263,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + key_rotate: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" kubernetes_apiserver: requests: memory: "128Mi" @@ -254,3 +288,4 @@ manifests: secret_ingress_tls: false service: true service_ingress: false + job_key_rotate: true