(apiserver) support key rotation

- Support key rotation for the etcd encryption key in the apiserver chart - Remove configmap annotations from the apiserver anchor pods as the pod is built to pickup changes in configmap contents without restart. - Also update the apiserver anchor DaemonSet to apps/v1 and make required updates to support that update. Change-Id: I2d18996bbe04bada9da2bce01a502550d3681c97
2019-01-19 13:03:59 -06:00 · 2019-01-19 13:03:59 -06:00 · ad30aa7382
parent 105fa608d7
commit ad30aa7382
6 changed files with 300 additions and 27 deletions
--- a/charts/apiserver/templates/bin/_key_rotate.tpl
+++ b/charts/apiserver/templates/bin/_key_rotate.tpl
@ -0,0 +1,85 @@
+#!/bin/bash
+# Copyright 2019 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -ex
+
+TEMP_DIR=$(mktemp -d)
+ANNOTATION_NAME="airshipit.org/encryption_key"
+
+get_service_endpoints() {
+  ns="$1"
+  svc="$2"
+  kubectl -n $ns get endpoints -o json $svc | jq '.subsets[0].addresses' | jq '.[] | .targetRef.name' -r
+
+}
+
+get_pod_annotation() {
+  ns="$1"
+  pod_name="$2"
+  kubectl -n $ns get pod "$pod_name" -o json | jq ".metadata.annotations.\"${ANNOTATION_NAME}\""
+}
+
+get_annotations_key() {
+  echo $ENCRYPTION_ANNOTATION | tr -d ' ' | awk -F':' '{print $1}'
+}
+
+get_encryption_hash() {
+  echo $ENCRYPTION_ANNOTATION | tr -d ' ' | awk -F':' '{print $2}'
+}
+
+apiserver_compare() {
+  echo "${apiservers[@]}" | sort | uniq > "${TEMP_DIR}/a.txt"
+  echo "${updated_apiservers[@]}" | sort | uniq > "${TEMP_DIR}/b.txt"
+  comm -3 "${TEMP_DIR}/a.txt" "${TEMP_DIR}/b.txt"
+}
+
+{{- $envAll := . }}
+
+
+{{- if and (.Values.conf) (hasKey .Values.conf "encryption_provider") }}
+
+ENCRYPTION_ANNOTATION='{{ $envAll | include "kubernetes_apiserver.key_annotation" }}'
+KUBE_SERVICE_NAMESPACE=${KUBE_SERVICE_NAMESPACE:-"kube-system"}
+KUBE_SERVICE_NAME=${KUBE_SERVICE_NAME:-"kubernetes-apiserver"}
+
+apiservers=( $(get_service_endpoints "$KUBE_SERVICE_NAMESPACE" "$KUBE_SERVICE_NAME"))
+updated_apiservers=()
+
+annotation="$(get_annotations_key)"
+
+# TODO(sh8121att) add timeout logic
+while [[ -n "$(apiserver_compare)" ]];
+do
+  for pod_name in "${apiservers[@]}";
+  do
+    pod_key=$(get_pod_annotation "$KUBE_SERVICE_NAMESPACE" "$pod_name")
+    if [ "$pod_key" == "$(get_encryption_hash)" ];
+    then
+      updated_apiservers+=("$pod_name")
+    fi
+  done
+done
+
+echo "All apiserver instances have an updated key."
+
+while true
+do
+  kubectl get secrets --all-namespaces -o json | kubectl replace --validate=false -f -
+  if [[ $? -eq 0 ]]
+  then
+    echo "All secret resources re-encrypted."
+    exit 0
+  fi
+done
+{{- end -}}
--- a/charts/apiserver/templates/configmap-bin.yaml
+++ b/charts/apiserver/templates/configmap-bin.yaml
@ -27,4 +27,6 @@ data:
 {{ tuple "bin/_anchor.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
  pre_stop: |+
 {{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  key-rotate.sh: |+
+{{ tuple "bin/_key_rotate.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
--- a/charts/apiserver/templates/daemonset.yaml
+++ b/charts/apiserver/templates/daemonset.yaml
@ -33,15 +33,17 @@ spec:
    matchLabels:
 {{ $labels | indent 6 }}
 {{ tuple $envAll "kubernetes-apiserver-anchor" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
+  selector:
+    matchLabels:
+{{ $labels | indent 6 }}
  template:
    metadata:
      labels:
 {{ $labels | indent 8 }}
      annotations:
+        {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
        scheduler.alpha.kubernetes.io/critical-pod: ''
        configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
-        configmap-certs-hash: {{ tuple "configmap-certs.yaml" . | include "helm-toolkit.utils.hash" }}
-        configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
    spec:
      nodeSelector:
        {{ .Values.labels.kubernetes_apiserver.node_selector_key }}: {{ .Values.labels.kubernetes_apiserver.node_selector_value }}
@ -63,8 +65,8 @@ spec:
            value: /host{{ .Values.anchor.kubelet.manifest_path }}/{{ .Values.service.name }}.yaml
          - name: PKI_PATH
            value: /host{{ .Values.apiserver.host_etc_path }}/pki
-          command:
-            - /tmp/bin/anchor
+          command: ["/bin/sh","-c"]
+          args: ["/tmp/bin/anchor"]
          lifecycle:
            preStop:
              exec:
--- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
+++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl
@ -12,6 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+{{- define "kubernetes_apiserver.key_concat" -}}
+    {{- range $encProv := .Values.conf.encryption_provider.content.resources -}}
+      {{- if hasKey (index $encProv "providers" 0) "identity" -}}
+        {{- printf "%s" (index $encProv "providers" 0 "identity") -}}
+      {{- else if hasKey (index $encProv "providers" 0) "secretbox" -}}
+        {{- printf "%s" (index $encProv "providers" 0 "secretbox" "keys" 0 "secret") -}}
+      {{- else if hasKey (index $encProv "providers" 0) "aescbc" -}}
+        {{- printf "%s" (index $encProv "providers" 0 "aescbc" "keys" 0 "secret") -}}
+      {{- else if hasKey (index $encProv "providers" 0) "aesgcm" -}}
+        {{- printf "%s" (index $encProv "providers" 0 "aesgcm" "keys" 0 "secret") -}}
+      {{- end -}}
+    {{- end -}}
+{{- end -}}
+
+
+{{- define "kubernetes_apiserver.key_annotation" -}}
+  {{- if and (.Values.conf) (hasKey .Values.conf "encryption_provider") -}}
+    {{- $encKey := ( . | include "kubernetes_apiserver.key_concat") -}}
+{{ .Values.const.encryption_annotation | quote }}: {{ sha256sum $encKey | quote }}
+  {{- end -}}
+{{- end -}}
+
+
 {{- $envAll := . }}
 ---
 apiVersion: v1
@ -23,6 +46,7 @@ metadata:
    {{ .Values.service.name }}-service: enabled
 {{ tuple $envAll "kubernetes" "apiserver" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
  annotations:
+    {{ $envAll | include "kubernetes_apiserver.key_annotation" }}
    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
 spec:
  hostNetwork: true
--- a/charts/apiserver/templates/job-key-rotate.yaml
+++ b/charts/apiserver/templates/job-key-rotate.yaml
@ -0,0 +1,125 @@
+{{/*
+Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_key_rotate }}
+{{- $envAll := . }}
+{{ $serviceAccountName := "apiserver-key-rotate" }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ $serviceAccountName }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: {{ $serviceAccountName }}-cluster
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - list
+      - get
+      - delete
+      - create
+      - update
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $serviceAccountName }}-cluster
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}-cluster
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}-{{ $envAll.Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - pods
+    verbs:
+      - get
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}-{{ $envAll.Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  namespace: {{ $envAll.Release.Namespace }}
+  name: {{ $serviceAccountName }}-{{ $envAll.Release.Namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: apiserver-key-rotate
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  template:
+    metadata:
+      annotations:
+        {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+      labels:
+{{ tuple $envAll "kube-apiserver" "key-rotate" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "key_rotate" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: apiserver-key-rotate
+          image: {{ .Values.images.tags.key_rotate | quote }}
+          imagePullPolicy: {{ .Values.images.pull_policy | quote }}
+{{ tuple $envAll $envAll.Values.pod.resources.key_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: "ANNOTATION_NAME"
+              value: {{ .Values.const.encryption_annotation | quote }}
+          command:
+            - /tmp/key-rotate.sh
+          volumeMounts:
+            - name: apiserver-bin
+              mountPath: /tmp/key-rotate.sh
+              subPath: key-rotate.sh
+              readOnly: true
+      volumes:
+        - name: apiserver-bin
+          configMap:
+            name: {{ .Values.service.name }}-bin
+            defaultMode: 0555
+...
+{{- end }}
--- a/charts/apiserver/values.yaml
+++ b/charts/apiserver/values.yaml
@ -17,6 +17,7 @@ release_group: null
 # NOTE(mark-burnett): These values are not really configurable -- they live
 # here to keep the templates cleaner.
 const:
+  encryption_annotation: "airshipit.org/encryption_key"
  command_prefix:
    - /apiserver
    - --advertise-address=$(POD_IP)
@ -55,14 +56,24 @@ const:

 images:
  tags:
+    dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
    anchor: gcr.io/google_containers/hyperkube-amd64:v1.11.6
    apiserver: gcr.io/google_containers/hyperkube-amd64:v1.11.6
+    key_rotate: gcr.io/google_containers/hyperkube-amd64:v1.11.6
  pull_policy: "IfNotPresent"
+  local_registry:
+    active: false
+    exclude:
+      - dep_check
+      - image_repo_sync

 labels:
  kubernetes_apiserver:
    node_selector_key: kubernetes-apiserver
    node_selector_value: enabled
+  job:
+    node_selector_key: kubernetes-apiserver
+    node_selector_value: enabled

 anchor:
  dns_policy: Default
@ -85,7 +96,17 @@ conf:
 #      plugins:
 #        - name: EventRateLimit
 #          path: eventconfig.yaml
-#
+#  eventconfig:
+#    file: eventconfig.yaml
+#    command_options: []
+#    content:
+#      kind: Configuration
+#      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+#      limits:
+#        - type: Server
+#          qps: 1000
+#          burst: 10000
+
 # Uncomment any of the below to enable enhanced Audit Logging command line options.
 #
 #  auditpolicy:
@ -98,28 +119,22 @@ conf:
 #       rules:
 #         - level: Metadata
 #
-#  eventconfig:
-#    file: eventconfig.yaml
-#    command_options:
-#      - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
-#    content:
-#      kind: Configuration
-#      apiVersion: eventratelimit.admission.k8s.io/v1alpha1
-#      limits:
-#        - type: Server
-#          qps: 1000
-#          burst: 10000
-#  encryption_provider:
-#    file: encryption_provider.yaml
-#    command_option: ''
-#    content:
-#      kind: EncryptionConfig
-#      apiVersion: v1
-#      resources:
-#        - resources:
-#            - 'secrets'
-#          providers:
-#            - identity: {}
+  encryption_provider:
+    file: encryption_provider.yaml
+    command_options:
+      - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
+    content:
+      kind: EncryptionConfig
+      apiVersion: v1
+      resources:
+        - resources:
+            - 'secrets'
+          providers:
+            - secretbox:
+                keys:
+                  - name: key1
+                    secret: Xw2UcbjILTJM6QiFZ0WPSbUvjtoT8OJC/Nl8qqYWjGk=
+            - identity: {}

 apiserver:
  arguments:
@ -181,6 +196,18 @@ secrets:
      cert: null
      key: null

+dependencies:
+  dynamic:
+    common:
+      local_image_registry:
+        jobs:
+          - apiserver-image-repo-sync
+        services:
+          - endpoint: node
+            service: local_image_registry
+  static:
+    key_rotate: {}
+
 # typically overriden by environmental
 # values, but should include all endpoints
 # required by this chart
@ -236,6 +263,13 @@ pod:
      limits:
        memory: "1024Mi"
        cpu: "2000m"
+    key_rotate:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
    kubernetes_apiserver:
      requests:
        memory: "128Mi"
@ -254,3 +288,4 @@ manifests:
  secret_ingress_tls: false
  service: true
  service_ingress: false
+  job_key_rotate: true