From c13fc33d85b451d0edb173691994642b03e596fc Mon Sep 17 00:00:00 2001
From: Scott Hussey <sh8121@att.com>
Date: Fri, 14 Dec 2018 12:49:22 -0600
Subject: [PATCH] Support systemd unit management during node join

- Support systemctl enable/start/stop/disable commands during join.sh
  or genesis.sh

Change-Id: I28046afbc55fc1d1af4575778f614f928f0e91c9
---
 examples/basic/HostSystem.yaml    | 34 +++++++++++++++++++++++
 examples/basic/Kubelet.yaml       |  1 +
 promenade/config.py               | 45 +++++++++++++++++++++++++++++++
 promenade/schemas/HostSystem.yaml | 17 +++++++++++-
 promenade/templates/include/up.sh |  7 +++++
 5 files changed, 103 insertions(+), 1 deletion(-)

diff --git a/examples/basic/HostSystem.yaml b/examples/basic/HostSystem.yaml
index 728a3ce2..2f4cbb0a 100644
--- a/examples/basic/HostSystem.yaml
+++ b/examples/basic/HostSystem.yaml
@@ -8,11 +8,45 @@ metadata:
     layer: site
   storagePolicy: cleartext
 data:
+  systemd_units:
+    kube-cgroup:
+      enable: true
   files:
     - path: /opt/kubernetes/bin/kubelet
       tar_url: https://dl.k8s.io/v1.10.2/kubernetes-node-linux-amd64.tar.gz
       tar_path: kubernetes/node/bin/kubelet
       mode: 0555
+    - path: /etc/systemd/system/kube-cgroup.service
+      content: |
+        [Unit]
+        Description=Create and tune cgroup for Kubernetes Pods
+        Requires=network-online.target
+        Before=kubelet.service
+
+        [Service]
+        Delegate=yes
+        ExecStart=/usr/local/sbin/kube-cgroup.sh
+
+        [Install]
+        RequiredBy=kubelet.service
+      mode: 0444
+    - path: /usr/local/sbin/kube-cgroup.sh
+      mode: 0744
+      content: |-
+        #!/bin/bash
+
+        set -x
+
+        KUBE_CGROUP=${KUBE_CGROUP:-"kube_whitelist"}
+        SYSTEMD_ABSPATH="/sys/fs/cgroup/systemd/$KUBE_CGROUP"
+        CPUSET_ABSPATH="/sys/fs/cgroup/cpuset/$KUBE_CGROUP"
+        CPU_ABSPATH="/sys/fs/cgroup/cpu/$KUBE_CGROUP"
+        MEM_ABSPATH="/sys/fs/cgroup/memory/$KUBE_CGROUP"
+
+        for cg in $SYSTEMD_ABSPATH $CPUSET_ABSPATH $CPU_ABSPATH $MEM_ABSPATH
+        do
+          mkdir -p "$cg"
+        done
     - path: /etc/logrotate.d/json-logrotate
       mode: 0444
       content: |-
diff --git a/examples/basic/Kubelet.yaml b/examples/basic/Kubelet.yaml
index 60074cd3..69c5fa75 100644
--- a/examples/basic/Kubelet.yaml
+++ b/examples/basic/Kubelet.yaml
@@ -18,6 +18,7 @@ data:
     - --anonymous-auth=false
     - --feature-gates=PodShareProcessNamespace=true
     - --v=3
+    - --cgroup-root=/kube_whitelist
   images:
     pause: gcr.io/google_containers/pause-amd64:3.0
 ...
diff --git a/promenade/config.py b/promenade/config.py
index 296fec77..79febba1 100644
--- a/promenade/config.py
+++ b/promenade/config.py
@@ -169,6 +169,51 @@ class Configuration:
             if value:
                 return value
 
+    @property
+    def enable_units(self):
+        """ Get systemd unit names where enable is ``true``."""
+        return self.get_units_by_action('enable')
+
+    @property
+    def start_units(self):
+        """ Get systemd unit names where start is ``true``."""
+        return self.get_units_by_action('start')
+
+    @property
+    def stop_units(self):
+        """ Get systemd unit names where stop is ``true``."""
+        return self.get_units_by_action('stop')
+
+    @property
+    def disable_units(self):
+        """ Get systemd unit names where disable is ``true``."""
+        return self.get_units_by_action('disable')
+
+    def get_units_by_action(self, action):
+        """ Select systemd unit names by ``action``
+
+        Get all units that are ``true`` for ``action``.
+        """
+        return [
+            k for k, v in self.systemd_units.items() if v.get(action, False)
+        ]
+
+    @property
+    def systemd_units(self):
+        """ Return a dictionary of systemd units to be managed during join.
+
+        The dictionary key is the systemd unit name, each will have a four
+        boolean keys: ``enable``, ``disable``, ``start``, ``stop`` on the
+        actions to be taken at the end of genesis/node join. The steps
+        are ordered: enable, start, stop, disable.
+        """
+        all_units = {}
+
+        for document in self.iterate(kind='HostSystem'):
+            all_units.update(document['data'].get('systemd_units', {}))
+
+        return all_units
+
     @property
     def join_ips(self):
         maybe_ips = self.get_path('KubernetesNode:join_ips')
diff --git a/promenade/schemas/HostSystem.yaml b/promenade/schemas/HostSystem.yaml
index fe2a9432..45edb0e5 100644
--- a/promenade/schemas/HostSystem.yaml
+++ b/promenade/schemas/HostSystem.yaml
@@ -11,6 +11,18 @@ data:
     abs_path:
       type: string
       pattern: '^/.+$'
+    systemd_unit:
+      type: object
+      properties:
+        enable:
+          type: boolean
+        disable:
+          type: boolean
+        start:
+          type: boolean
+        stop:
+          type: boolean
+      additionalProperties: false
     apt_source_line:
       type: string
       # XXX add regex
@@ -27,7 +39,6 @@ data:
           $ref: '#/definitions/url'
         tar_path:
           $ref: '#/definitions/rel_path'
-
       requried:
         - mode
         - path
@@ -68,6 +79,10 @@ data:
         type: object
         items:
           $ref: '#/definitions/file'
+    systemd_units:
+      type: object
+      additionalProperties:
+        $ref: '#/definitions/systemd_unit'
     images:
       type: object
       properties:
diff --git a/promenade/templates/include/up.sh b/promenade/templates/include/up.sh
index b0eb2295..613291df 100644
--- a/promenade/templates/include/up.sh
+++ b/promenade/templates/include/up.sh
@@ -105,6 +105,13 @@ log
 log === Starting Docker and Kubelet ===
 set -x
 systemctl daemon-reload
+
+{% for a in ['enable','start','stop','disable'] %}
+{% for u in config.get_units_by_action(a) %}
+systemctl {{ a }} {{ u }}
+{% endfor %}
+{% endfor %}
+
 systemctl restart docker || true
 systemctl enable kubelet
 systemctl restart kubelet