apiserver(-webhook): Allow volume overrides

The existing apiserver chart supports volume overrides for the anchor
daemonset, but not for the apiserver static pod itself. The feature to
allow volume overrides in the apiserver-webhook chart was never fully
implemented.

This changes allows volume overrides via values.yaml for both charts,
and provides a more complete audit example that includes mounting the
audit log destination as a host path volume.

Change-Id: I27ccf77671a190e8cb6b66d8a9b13c2cde6c9a45

charts/apiserver-webhook/templates/deployment.yaml

@@ -96,6 +96,8 @@ tls-{{ $group | replace "_" "-" }}-{{ $type | replace "_" "-" }}
 {{- end -}}
 
 {{- if .Values.manifests.deployment }}
+{{- $mounts_apiserver := .Values.pod.mounts.apiserver_webhook.apiserver }}
+{{- $mounts_webhook := .Values.pod.mounts.apiserver_webhook.webhook }}
 {{- $envAll := . }}
 ---
 apiVersion: apps/v1
@@ -212,6 +214,7 @@ spec:
 {{ tuple "kubelet" "client" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
 {{ tuple "etcd" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
 {{ tuple "etcd" "client" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
+{{ if $mounts_apiserver.volumeMounts }}{{ toYaml $mounts_apiserver.volumeMounts | indent 12 }}{{ end }}
         - name: webhook
 {{ tuple $envAll "kubernetes_keystone_webhook" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@@ -248,6 +251,7 @@ spec:
               subPath: webhook_start.sh
               readOnly: true
 {{ tuple "keystone_webhook" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
+{{ if $mounts_webhook.volumeMounts }}{{ toYaml $mounts_webhook.volumeMounts | indent 12 }}{{ end }}
       volumes:
 {{- if hasKey .Values.certificates "keystone" }}
 {{ tuple "keystone" "server" $envAll.Values.conf.paths.pki $envAll | include "local.mount_cert_bundle" | indent 12 }}
@@ -277,4 +281,6 @@ spec:
           secret:
             defaultMode: 292
             secretName: {{ .Values.secrets.tls.webhook_apiserver.api.public }}
+{{ if $mounts_apiserver.volumes }}{{ toYaml $mounts_apiserver.volumes | indent 8 }}{{ end }}
+{{ if $mounts_webhook.volumes }}{{ toYaml $mounts_webhook.volumes | indent 8 }}{{ end }}
 {{- end }}

charts/apiserver-webhook/values.yaml

@@ -214,13 +214,26 @@ pod:
         apiserver:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
+          # explicitly setting the runAsUser may be required to write audit logs to the host
+          # runAsUser: 0
         webhook:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
   mounts:
-    kubernetes_apiserver:
-      init_container: null
-      kubernetes_apiserver:
+    apiserver_webhook:
+      apiserver:
+# Example mounts for audit logging, refer to .conf.apiserver.auditpolicy below.
+#        volumeMounts:
+#          - name: audit-logs
+#            mountPath: /var/log/audit
+#            mountPropagation: HostToContainer
+#            readOnly: false
+#        volumes:
+#          - name: audit-logs
+#            hostPath:
+#              path: /var/log/audit
+#              type: DirectoryOrCreate
+      webhook: null
   affinity:
     anti:
       type:
@@ -282,13 +295,6 @@ pod:
         limits:
           memory: "256Mi"
           cpu: "200m"
-  mounts:
-    kubernetes_keystone_webhook_api:
-      init_container: null
-      kubernetes_keystone_webhook_api: null
-    kubernetes_keystone_webhook_tests:
-      init_container: null
-      kubernetes_keystone_webhook_tests: null
 conf:
   paths:
     base: '/etc/webhook_apiserver/'
@@ -330,6 +336,24 @@ conf:
       content:
         kind: EncryptionConfiguration
         apiVersion: apiserver.config.k8s.io/v1
+# Uncomment any of the below to enable enhanced Audit Logging command line options.
+# Note: To use the Log backend, ensure that the hostPath of the log file is mounted,
+# and that the runAsUser for the apiserver container can write to it.
+# (Refer to .pod.mounts.apiserver.apiserver)
+#
+#    auditpolicy:
+#      file: audit_policy.yaml
+#      command_options:
+#         - '--audit-policy-file=/etc/kubernetes/apiserver/audit_policy.yaml'
+#         - '--audit-log-maxsize=10'
+#         - '--audit-log-maxbackup=3'
+#         - '--audit-log-path=/var/log/audit/webhook-audit.log'
+#      content:
+#         kind: Policy
+#         apiVersion: apiserver.k8s.io/v1
+#         rules:
+#           - level: Metadata
+#
   policy:
     - resource:
         verbs:

charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl

@@ -51,6 +51,9 @@ httpGet:
 {{- end -}}
 
 
+{{- $mounts_apiserver := .Values.pod.mounts.apiserver.apiserver }}
+
+
 {{- $envAll := . }}
 ---
 apiVersion: v1
@@ -124,7 +127,9 @@ spec:
       volumeMounts:
         - name: etc
           mountPath: /etc/kubernetes/apiserver
+{{ if $mounts_apiserver.volumeMounts }}{{ toYaml $mounts_apiserver.volumeMounts | indent 8 }}{{ end }}
   volumes:
     - name: etc
       hostPath:
         path: {{ .Values.apiserver.host_etc_path }}
+{{ if $mounts_apiserver.volumes }}{{ toYaml $mounts_apiserver.volumes | indent 4 }}{{ end }}

charts/apiserver/values.yaml

@@ -131,11 +131,16 @@ conf:
 #    content: |
 #      -----SOME KEY-----
 # Uncomment any of the below to enable enhanced Audit Logging command line options.
+# Note: To use the Log backend, ensure that the hostPath of the log file is mounted.
+# (Refer to .pod.mounts.apiserver.apiserver)
 #
 #  auditpolicy:
 #    file: audit_policy.yaml
 #    command_options:
 #       - '--audit-policy-file=/etc/kubernetes/apiserver/audit_policy.yaml'
+#       - '--audit-log-maxsize=10'
+#       - '--audit-log-maxbackup=3'
+#       - '--audit-log-path=/var/log/audit/audit.log'
 #    content:
 #       kind: Policy
 #       apiVersion: apiserver.k8s.io/v1
@@ -293,9 +298,24 @@ pod:
           runAsUser: 0
           readOnlyRootFilesystem: false
   mounts:
+    # .pod.mounts.kubernetes_apiserver is for the anchor daemonset
     kubernetes_apiserver:
       init_container: null
       kubernetes_apiserver:
+    # .pod.mounts.apiserver is for the apiserver static pod
+    apiserver:
+      apiserver:
+# Example mounts for audit logging, refer to .conf.auditpolicy above.
+#        volumeMounts:
+#          - name: audit-logs
+#            mountPath: /var/log/audit
+#            mountPropagation: HostToContainer
+#            readOnly: false
+#        volumes:
+#          - name: audit-logs
+#            hostPath:
+#              path: /var/log/audit
+#              type: DirectoryOrCreate
   replicas:
     apiserver: 3
   lifecycle: