Migrate config to KubeletConfiguration

This patchset changes the way that kubelet receives it configuration
parameters so that we can enable [dynamic kubelet configuration][1] down
the line. Starting in Kubernetes v1.11 the configuration of some
parameters has been moved from command line arguments to a static
[configuration file][2].

[1] https://kubernetes.io/docs/tasks/administer-cluster/reconfigure-kubelet/
[2] https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/

Change-Id: Id406ae81fcf44ed0319513e5befc37fd4cff30e5

doc/source/configuration/kubelet.rst

@@ -2,9 +2,12 @@ Kubelet
 =======
 
 Configuration for the Kubernetes worker daemon (the Kubelet).  This document
-contains two keys: ``arguments`` and ``images``.  The ``arguments`` are
-appended directly to the ``kubelet`` command line, along with arguments that
-are controlled by Promenade more directly.
+contains three keys: ``arguments``, ``images``, and ``config_file_overrides``.
+The ``arguments`` are appended directly to the ``kubelet`` command line,
+along with arguments that are controlled by Promenade more directly.
+The ``config_file_overrides`` are appended directly to the static kubelet
+configuration file and only consists of a subset of kubelet arguments.
+More information regarding the format for this key can be found here_.
 
 The only image that is configurable is for the ``pause`` container.
 
@@ -27,9 +30,12 @@ Here is a sample document:
       arguments:
         - --cni-bin-dir=/opt/cni/bin
         - --cni-conf-dir=/etc/cni/net.d
-        - --eviction-max-pod-grace-period=-1
         - --network-plugin=cni
-        - --node-status-update-frequency=5s
         - --v=5
       images:
         pause: gcr.io/google_containers/pause-amd64:3.0
+      config_file_overrides:
+        evictionMaxPodGracePeriod: -1
+        nodeStatusUpdateFrequency: "5s"
+
+.. _here: https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file

examples/basic/Kubelet.yaml

@@ -11,14 +11,16 @@ data:
   arguments:
     - --cni-bin-dir=/opt/cni/bin
     - --cni-conf-dir=/etc/cni/net.d
-    - --eviction-max-pod-grace-period=-1
     - --network-plugin=cni
-    - --node-status-update-frequency=5s
-    - --serialize-image-pulls=false
-    - --anonymous-auth=false
-    - --feature-gates=PodShareProcessNamespace=true
     - --v=3
-    - --cgroup-root=/kube_whitelist
   images:
     pause: gcr.io/google_containers/pause-amd64:3.0
+  config_file_overrides:
+    cgroupRoot: "/kube_whitelist"
+    evictionMaxPodGracePeriod: -1
+    featureGates:
+      PodShareProcessNamespace: true
+      TaintBasedEvictions: false
+    nodeStatusUpdateFrequency: "5s"
+    serializeImagePulls: false
 ...

examples/complete/Kubelet.yaml

@@ -11,11 +11,12 @@ data:
   arguments:
     - --cni-bin-dir=/opt/cni/bin
     - --cni-conf-dir=/etc/cni/net.d
-    - --eviction-max-pod-grace-period=-1
     - --network-plugin=cni
-    - --node-status-update-frequency=5s
-    - --serialize-image-pulls=false
     - --v=5
   images:
     pause: gcr.io/google_containers/pause-amd64:3.0
+  config_file_overrides:
+    evictionMaxPodGracePeriod: -1
+    nodeStatusUpdateFrequency: "5s"
+    serializeImagePulls: false
 ...

examples/gate/Kubelet.yaml

@@ -11,13 +11,15 @@ data:
   arguments:
     - --cni-bin-dir=/opt/cni/bin
     - --cni-conf-dir=/etc/cni/net.d
-    - --eviction-max-pod-grace-period=-1
     - --network-plugin=cni
-    - --node-status-update-frequency=5s
-    - --serialize-image-pulls=false
-    - --anonymous-auth=false
-    - --feature-gates=PodShareProcessNamespace=true
     - --v=3
   images:
     pause: gcr.io/google_containers/pause-amd64:3.0
+  config_file_overrides:
+    evictionMaxPodGracePeriod: -1
+    featureGates:
+      PodShareProcessNamespace: true
+      TaintBasedEvictions: false
+    nodeStatusUpdateFrequency: "5s"
+    serializeImagePulls: false
 ...

promenade/schemas/Kubelet.yaml

@@ -26,6 +26,8 @@ data:
       type: array
       items:
         type: string
+    config_file_overrides:
+      type: object
   required:
     - images
   additionalProperties: false

promenade/templates/roles/common/etc/kubernetes/kubelet/config.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    enabled: true
+  x509:
+    clientCAFile: "/etc/kubernetes/pki/kubelet-client-ca.pem"
+authorization:
+  mode: AlwaysAllow
+clusterDNS:
+- {{ config['KubernetesNetwork:dns.service_ip'] }}
+clusterDomain: {{ config['KubernetesNetwork:dns.cluster_domain'] }}
+staticPodPath: "/etc/kubernetes/manifests"
+tlsCertFile: "/etc/kubernetes/pki/kubelet.pem"
+tlsPrivateKeyFile: "/etc/kubernetes/pki/kubelet-key.pem"
+{%- if config['Kubelet:config_file_overrides'] is defined %}
+{{ config.get_path('Kubelet:config_file_overrides') | toyaml }}
+{%- endif %}

promenade/templates/roles/common/etc/systemd/system/kubelet.service

@@ -5,16 +5,10 @@ After=network-online.target
 
 [Service]
 ExecStart=/opt/kubernetes/bin/kubelet \
-    --anonymous-auth=false \
-    --client-ca-file=/etc/kubernetes/pki/kubelet-client-ca.pem \
-    --cluster-dns={{ config['KubernetesNetwork:dns.service_ip'] }} \
-    --cluster-domain={{ config['KubernetesNetwork:dns.cluster_domain'] }} \
+    --config=/etc/kubernetes/kubelet/config.yaml \
     --hostname-override={{ config.get_first('Genesis:hostname', 'KubernetesNode:hostname') }} \
     --kubeconfig=/etc/kubernetes/kubeconfig \
     --node-ip={{ config.get_first('Genesis:ip', 'KubernetesNode:ip') }} \
-    --pod-manifest-path=/etc/kubernetes/manifests \
-    --tls-cert-file=/etc/kubernetes/pki/kubelet.pem \
-    --tls-private-key-file=/etc/kubernetes/pki/kubelet-key.pem \
     {%- if config['Genesis:labels.static'] is defined %}
     --node-labels={{ config['Genesis:labels.static'] | join(',') }} \
     {%- elif config['KubernetesNode:labels.static'] is defined %}

tests/unit/api/test_validatedesign.py

@@ -201,12 +201,15 @@ VALID_DOCS = [
         'data': {
             'arguments': [
                 '--cni-bin-dir=/opt/cni/bin', '--cni-conf-dir=/etc/cni/net.d',
-                '--eviction-max-pod-grace-period=-1', '--network-plugin=cni',
-                '--node-status-update-frequency=5s',
-                '--serialize-image-pulls=false', '--v=5'
+                '--network-plugin=cni', '--v=5'
             ],
             'images': {
                 'pause': 'gcr.io/google_containers/pause-amd64:3.0'
+            },
+            'config_file_overrides': {
+                'evictionMaxPodGracePeriod': -1,
+                'nodeStatusUpdateFrequency': '5s',
+                'serializeImagePulls': 'false'
             }
         },
         'metadata': {

tests/unit/builder_data/simple/Kubelet.yaml

@@ -11,11 +11,12 @@ data:
   arguments:
     - --cni-bin-dir=/opt/cni/bin
     - --cni-conf-dir=/etc/cni/net.d
-    - --eviction-max-pod-grace-period=-1
     - --network-plugin=cni
-    - --node-status-update-frequency=5s
-    - --serialize-image-pulls=false
     - --v=5
   images:
     pause: gcr.io/google_containers/pause-amd64:3.0
+  config_file_overrides:
+    evictionMaxPodGracePeriod: -1
+    nodeStatusUpdateFrequency: "5s"
+    serializeImagePulls: false
 ...

tools/gate/config-templates/site-config.yaml

@@ -120,11 +120,12 @@ data:
   arguments:
     - --cni-bin-dir=/opt/cni/bin
     - --cni-conf-dir=/etc/cni/net.d
-    - --eviction-max-pod-grace-period=-1
     - --network-plugin=cni
-    - --node-status-update-frequency=5s
-    - --serialize-image-pulls=false
     - --v=5
   images:
     pause: gcr.io/google_containers/pause-amd64:3.0
+  config_file_overrides:
+    evictionMaxPodGracePeriod: -1
+    nodeStatusUpdateFrequency: "5s"
+    serializeImagePulls: false
 ...