summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorskovaleff <sk607s@att.com>2018-10-22 18:13:59 -0700
committerskovaleff <sk607s@att.com>2018-11-01 09:09:48 -0700
commit7ed8c29f99d5acb51a63b6bdb7c4a5f86c9b4245 (patch)
treeca551248f2f7716e7f10a31ce3b82462819ebebe
parenta648dcb2db2b842656bfee08102081023080e4d5 (diff)
Add ability to control pam_limits via new module 'limits'
1) 'Values' configures limit settings to be persisted. 2) Previous DivingBell controlled limits those were set but now are gone are cleared. 3) Previous values of newly set limits are backed up to /var/divingbell/limits 4) New limit is applied via adding a separate conf file to /etc/security/limits.d 5) The Doc is updated with appropriate details. 6) Dev env with Vagrant 7) Increase number of expected DaemonSets in 020-test 8) Demo: https://asciinema.org/a/209619 Change-Id: I5efb39c498c2b666b4ba97271b59757f4a0c1ca7
Notes
Notes (review): Code-Review+2: Craig Anderson <craig.anderson@att.com> Workflow+1: Craig Anderson <craig.anderson@att.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Thu, 01 Nov 2018 18:00:43 +0000 Reviewed-on: https://review.openstack.org/612555 Project: openstack/airship-divingbell Branch: refs/heads/master
-rw-r--r--Vagrantfile29
-rw-r--r--divingbell/templates/bin/_limits.sh.tpl109
-rw-r--r--divingbell/templates/bin/_sysctl.sh.tpl2
-rw-r--r--divingbell/templates/configmap-limits.yaml30
-rw-r--r--divingbell/templates/daemonset-limits.yaml71
-rw-r--r--divingbell/values.yaml27
-rw-r--r--doc/source/index.rst36
-rwxr-xr-xtools/gate/scripts/020-test-divingbell.sh42
8 files changed, 340 insertions, 6 deletions
diff --git a/Vagrantfile b/Vagrantfile
new file mode 100644
index 0000000..9a840c8
--- /dev/null
+++ b/Vagrantfile
@@ -0,0 +1,29 @@
1# -*- mode: ruby -*-
2# vi: set ft=ruby :
3
4Vagrant.configure("2") do |config|
5 config.vm.box = "generic/ubuntu1604"
6
7 [:virtualbox, :parallels, :libvirt, :hyperv].each do |provider|
8 config.vm.provider provider do |vplh, override|
9 vplh.cpus = 4
10 vplh.memory = 4096
11 end
12 end
13
14 config.vm.synced_folder "./", "/root/deploy/airship-divingbell"
15
16 config.vm.define "dbtest" do |node|
17 node.vm.hostname = "dbtest"
18 node.vm.provision :shell, inline: <<-SHELL
19 #mkdir /root/deploy
20 #git clone git://git.openstack.org/openstack/airship-divingbell /root/deploy/airship-divingbell
21 git clone https://git.openstack.org/openstack/openstack-helm-infra /root/deploy/openstack-helm-infra
22 cd /root/deploy/openstack-helm-infra
23 ./tools/gate/devel/start.sh full
24 cd /root/deploy/airship-divingbell/
25 ./tools/gate/scripts/010-build-charts.sh
26 ./tools/gate/scripts/020-test-divingbell.sh
27 SHELL
28 end
29end
diff --git a/divingbell/templates/bin/_limits.sh.tpl b/divingbell/templates/bin/_limits.sh.tpl
new file mode 100644
index 0000000..3e8f4e3
--- /dev/null
+++ b/divingbell/templates/bin/_limits.sh.tpl
@@ -0,0 +1,109 @@
1#!/bin/bash
2
3{{/*
4# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17*/}}
18
19set -e
20
21cat <<'EOF' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/limits_host.sh
22{{ include "divingbell.shcommon" . }}
23
24fname_prefix='60-divingbell-'
25persist_path='/etc/security/limits.d'
26
27if [ ! -d "${persist_path}" ]; then
28 mkdir -p "${persist_path}"
29fi
30
31write_test "${persist_path}"
32
33add_limits_param(){
34 local limit="${1}"
35 die_if_null "${limit}" ", limit not supplied to function"
36 local domain="${2}"
37 die_if_null "${domain}" ", domain not supplied to function"
38 local type="${3}"
39 die_if_null "${type}" ", type not supplied to function"
40 local item="${4}"
41 die_if_null "${item}" ", item not supplied to function"
42 local value="${5}"
43 die_if_null "${value}" ", value not supplied to function"
44
45 file_content="${domain} ${type} ${item} ${value}"
46 file_name="${fname_prefix}${limit}.conf"
47 file_path="${persist_path}/${file_name}"
48
49 # Persist the new setting
50 if [ -f "${file_path}" ] &&
51 [ "$(cat ${file_path})" != "${file_content}" ] ||
52 [ ! -f "${file_path}" ]
53 then
54 echo "${file_content}" > "${file_path}"
55 log.INFO "Limits setting applied: ${file_content}"
56 else
57 log.INFO "No changes made to limits param: ${limit}"
58 fi
59
60 curr_settings="${curr_settings}${file_name}"$'\n'
61}
62
63{{- range $index, $limit := .Values.conf.limits }}
64add_limits_param {{ $index | squote }} {{ $limit.domain | squote }} {{ $limit.type | squote }}\
65 {{ $limit.item | squote }} {{ $limit.value | squote }}
66{{- end }}
67
68# Revert any previously applied limits settings which are now absent
69prev_files="$(find "${persist_path}" -type f)"
70if [ -n "${prev_files}" ]; then
71 basename -a ${prev_files} | sort > /tmp/prev_settings
72 echo "${curr_settings}" | sort > /tmp/curr_settings
73 revert_list="$(comm -23 /tmp/prev_settings /tmp/curr_settings)"
74 IFS=$'\n'
75 for orig_limits_setting in ${revert_list}; do
76 rm "${persist_path}/${orig_limits_setting}"
77 log.INFO "Reverted limits setting: ${persist_path}/${orig_limits_setting}"
78 done
79fi
80
81# Print limit settings
82# su is a simple and fast way to see applied changes
83# bash, bash -c, sudo, setsid didn't work out for me.
84su -c "prlimit --noheadings --output RESOURCE,SOFT,HARD"
85# The setting is persisted for a new process.
86# It's deliberate design decision to let current process be intact.
87# For this test it's just test bash process.
88# For production case it's limits_host.sh run by DivingBell pod which is in sleep mode.
89
90if [ -n "${curr_settings}" ]; then
91 log.INFO 'All limits configuration successfully validated on this node.'
92else
93 log.WARN 'No limits overrides defined for this node.'
94fi
95
96exit 0
97EOF
98
99chmod 755 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/limits_host.sh
100chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/limits_host.sh
101
102sleep 1
103echo 'INFO Putting the daemon to sleep.'
104
105while [ 1 ]; do
106 sleep 300
107done
108
109exit 0
diff --git a/divingbell/templates/bin/_sysctl.sh.tpl b/divingbell/templates/bin/_sysctl.sh.tpl
index 4c60c30..5a6d0d1 100644
--- a/divingbell/templates/bin/_sysctl.sh.tpl
+++ b/divingbell/templates/bin/_sysctl.sh.tpl
@@ -120,7 +120,7 @@ fi
120if [ -n "${curr_settings}" ]; then 120if [ -n "${curr_settings}" ]; then
121 log.INFO 'All sysctl configuration successfully validated on this node.' 121 log.INFO 'All sysctl configuration successfully validated on this node.'
122else 122else
123 log.WARN 'No syctl overrides defined for this node.' 123 log.WARN 'No sysctl overrides defined for this node.'
124fi 124fi
125 125
126exit 0 126exit 0
diff --git a/divingbell/templates/configmap-limits.yaml b/divingbell/templates/configmap-limits.yaml
new file mode 100644
index 0000000..d79c59f
--- /dev/null
+++ b/divingbell/templates/configmap-limits.yaml
@@ -0,0 +1,30 @@
1{{/*
2Copyright 2018 The Openstack-Helm Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/}}
16
17{{- define "divingbell.configmap.limits" }}
18{{- $configMapName := index . 0 }}
19{{- $envAll := index . 1 }}
20{{- with $envAll }}
21---
22apiVersion: v1
23kind: ConfigMap
24metadata:
25 name: {{ $configMapName }}
26data:
27 limits: |+
28{{ tuple "bin/_limits.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
29{{- end }}
30{{- end }}
diff --git a/divingbell/templates/daemonset-limits.yaml b/divingbell/templates/daemonset-limits.yaml
new file mode 100644
index 0000000..fd239d3
--- /dev/null
+++ b/divingbell/templates/daemonset-limits.yaml
@@ -0,0 +1,71 @@
1{{/*
2# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15*/}}
16
17{{- define "divingbell.daemonset.limits" }}
18 {{- $daemonset := index . 0 }}
19 {{- $configMapName := index . 1 }}
20 {{- $envAll := index . 2 }}
21 {{- with $envAll }}
22---
23apiVersion: extensions/v1beta1
24kind: DaemonSet
25metadata:
26 name: {{ $daemonset }}
27 annotations:
28 {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
29spec:
30{{ tuple $envAll $daemonset | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
31 template:
32 metadata:
33 labels:
34{{ list $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
35 spec:
36 hostNetwork: true
37 hostPID: true
38 hostIPC: true
39 containers:
40 - name: {{ $daemonset }}
41 image: {{ .Values.images.divingbell }}
42 imagePullPolicy: {{ .Values.images.pull_policy }}
43{{ tuple $envAll $envAll.Values.pod.resources.limits | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
44 command:
45 - /tmp/{{ $daemonset }}.sh
46 volumeMounts:
47 - name: rootfs-{{ $daemonset }}
48 mountPath: {{ .Values.conf.chroot_mnt_path }}
49 - name: {{ $configMapName }}
50 mountPath: /tmp/{{ $daemonset }}.sh
51 subPath: {{ $daemonset }}
52 readOnly: true
53 securityContext:
54 privileged: true
55 volumes:
56 - name: rootfs-{{ $daemonset }}
57 hostPath:
58 path: /
59 - name: {{ $configMapName }}
60 configMap:
61 name: {{ $configMapName }}
62 defaultMode: 0555
63 {{- end }}
64{{- end }}
65{{- if .Values.manifests.daemonset_limits }}
66{{- $daemonset := "limits" }}
67{{- $configMapName := "divingbell-limits" }}
68{{- $daemonset_yaml := list $daemonset $configMapName . | include "divingbell.daemonset.limits" | toString | fromYaml }}
69{{- $configmap_include := "divingbell.configmap.limits" }}
70{{- list $daemonset $daemonset_yaml $configmap_include $configMapName . | include "helm-toolkit.utils.daemonset_overrides" }}
71{{- end }}
diff --git a/divingbell/values.yaml b/divingbell/values.yaml
index f29e8e4..8436524 100644
--- a/divingbell/values.yaml
+++ b/divingbell/values.yaml
@@ -25,6 +25,21 @@ conf:
25 chroot_mnt_path: '/mnt' 25 chroot_mnt_path: '/mnt'
26 log_colors: False 26 log_colors: False
27 27
28## data.values.conf.sysctl
29# sysctl:
30# fs.suid_dumpable: '0'
31## data.values.conf.limits
32# limits:
33# nofile:
34# domain: 'root'
35# type: 'soft'
36# item: 'nofile'
37# value: '101'
38# core_dump:
39# domain: '0:'
40# type: 'hard'
41# item: 'core'
42# value: 0
28pod: 43pod:
29 lifecycle: 44 lifecycle:
30 upgrades: 45 upgrades:
@@ -46,6 +61,10 @@ pod:
46 enabled: true 61 enabled: true
47 min_ready_seconds: 0 62 min_ready_seconds: 0
48 max_unavailable: 100% 63 max_unavailable: 100%
64 limits:
65 enabled: true
66 min_ready_seconds: 0
67 max_unavailable: 100%
49 resources: 68 resources:
50 enabled: false 69 enabled: false
51 ethtool: 70 ethtool:
@@ -76,9 +95,17 @@ pod:
76 requests: 95 requests:
77 memory: "128Mi" 96 memory: "128Mi"
78 cpu: "100m" 97 cpu: "100m"
98 limits:
99 limits:
100 memory: "128Mi"
101 cpu: "100m"
102 requests:
103 memory: "128Mi"
104 cpu: "100m"
79 105
80manifests: 106manifests:
81 daemonset_ethtool: true 107 daemonset_ethtool: true
82 daemonset_mounts: true 108 daemonset_mounts: true
83 daemonset_uamlite: true 109 daemonset_uamlite: true
84 daemonset_sysctl: true 110 daemonset_sysctl: true
111 daemonset_limits: true
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 65d0b80..df01788 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -49,7 +49,8 @@ In order to keep configuration as isolated as possible from other systems that
49manage common files like /etc/fstab and /etc/sysctl.conf, Divingbell daemonsets 49manage common files like /etc/fstab and /etc/sysctl.conf, Divingbell daemonsets
50manage all of their configuration in separate files (e.g. by writing unique 50manage all of their configuration in separate files (e.g. by writing unique
51files to /etc/sysctl.d or defining unique Systemd units) to avoid potential 51files to /etc/sysctl.d or defining unique Systemd units) to avoid potential
52conflicts. 52conflicts. Another example is limit management, Divingbell daemonset writes
53separate files to /etc/security/limits.d.
53 54
54To maximize robustness and utility, the daemonsets in this chart are made to be 55To maximize robustness and utility, the daemonsets in this chart are made to be
55idempotent. In addition, they are designed to implicitly restore the original 56idempotent. In addition, they are designed to implicitly restore the original
@@ -78,6 +79,27 @@ Used to manage host level sysctl tunables. Ex::
78 net/ipv4/ip_forward: 1 79 net/ipv4/ip_forward: 1
79 net/ipv6/conf/all/forwarding: 1 80 net/ipv6/conf/all/forwarding: 1
80 81
82limits
83^^^^^^
84
85Used to manage host level limits. Ex::
86
87 conf:
88 limits:
89 nofile:
90 domain: 'root'
91 type: 'soft'
92 item: 'nofile'
93 value: '101'
94 core_dump:
95 domain: '0:'
96 type: 'hard'
97 item: 'core'
98 value: 0
99
100Previous values of newly set limits are backed up to /var/divingbell/limits
101
102
81mounts 103mounts
82^^^^^^ 104^^^^^^
83 105
@@ -256,6 +278,18 @@ Caveats:
256 "another_label" would take precedence and be applied to nodes that 278 "another_label" would take precedence and be applied to nodes that
257 contained both of the defined labels. 279 contained both of the defined labels.
258 280
281Dev Environment with Vagrant
282----------------------------
283The point of Dev env to prepare working environment for development.
284
285Vagrantfile allows to run on working copy with modifications
286e.g. to 020-test script. The approach is to setup Gate test
287but do not delete the pods and other stuff. You have:
288
2891. test run of previous tests and their results
2902. your changes from working tree are applied smoothly
2913. your not committed test runs in prepared env
292
259Recorded Demo 293Recorded Demo
260------------- 294-------------
261 295
diff --git a/tools/gate/scripts/020-test-divingbell.sh b/tools/gate/scripts/020-test-divingbell.sh
index c94adb6..9042ec6 100755
--- a/tools/gate/scripts/020-test-divingbell.sh
+++ b/tools/gate/scripts/020-test-divingbell.sh
@@ -57,7 +57,7 @@ for line in ${nic_info}; do
57 fi 57 fi
58 if [ "${physical_nic}" = 'true' ] && [[ ${line} = *'logical name'* ]]; then 58 if [ "${physical_nic}" = 'true' ] && [[ ${line} = *'logical name'* ]]; then
59 DEVICE="$(echo "${line}" | cut -d':' -f2 | tr -d '[:space:]')" 59 DEVICE="$(echo "${line}" | cut -d':' -f2 | tr -d '[:space:]')"
60 echo "Found deivce: '${DEVICE}' to use for ethtool testing" 60 echo "Found device: '${DEVICE}' to use for ethtool testing"
61 break 61 break
62 fi 62 fi
63done 63done
@@ -99,6 +99,7 @@ _teardown_systemd(){
99clean_persistent_files(){ 99clean_persistent_files(){
100 sudo rm -r /var/${NAME} >& /dev/null || true 100 sudo rm -r /var/${NAME} >& /dev/null || true
101 sudo rm -r /etc/sysctl.d/60-${NAME}-* >& /dev/null || true 101 sudo rm -r /etc/sysctl.d/60-${NAME}-* >& /dev/null || true
102 sudo rm -r /etc/security/limits.d/60-${NAME}-* >& /dev/null || true
102 _teardown_systemd ${MOUNTS_PATH1} mount 103 _teardown_systemd ${MOUNTS_PATH1} mount
103 _teardown_systemd ${MOUNTS_PATH2} mount 104 _teardown_systemd ${MOUNTS_PATH2} mount
104 _teardown_systemd ${MOUNTS_PATH3} mount 105 _teardown_systemd ${MOUNTS_PATH3} mount
@@ -312,6 +313,38 @@ test_sysctl(){
312 echo '[SUCCESS] sysctl test5 passed successfully' >> "${TEST_RESULTS}" 313 echo '[SUCCESS] sysctl test5 passed successfully' >> "${TEST_RESULTS}"
313} 314}
314 315
316_test_limits_value(){
317 local limit=${1}
318 local domain=${2}
319 local type=${3}
320 local item=${4}
321 local value=${5}
322 test "$(cat /etc/security/limits.d/60-${NAME}-${limit}.conf)" = \
323 "$domain $type $item $value"
324}
325
326test_limits(){
327 local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}.yaml
328 echo "conf:
329 limits:
330 limit1:
331 domain: root
332 type: hard
333 item: core
334 value: 0
335 limit2:
336 domain: '0:'
337 type: soft
338 item: nofile
339 value: 101" > "${overrides_yaml}"
340 echo $(cat ${overrides_yaml})
341 install_base "--values=${overrides_yaml}"
342 get_container_status limits
343 _test_limits_value limit1 root hard core 0
344 _test_limits_value limit2 '0:' soft nofile 101
345 echo "[SUCCESS] test range loop for limits passed successfully" >> "${TEST_RESULTS}"
346}
347
315_test_if_mounted_positive(){ 348_test_if_mounted_positive(){
316 mountpoint "${1}" || (echo "Expect ${1} to be mounted, but was not"; exit 1) 349 mountpoint "${1}" || (echo "Expect ${1} to be mounted, but was not"; exit 1)
317 df -h | grep "${1}" | grep "${2}" || 350 df -h | grep "${1}" | grep "${2}" ||
@@ -815,9 +848,9 @@ test_overrides(){
815 848
816 # Compare against expected number of generated daemonsets 849 # Compare against expected number of generated daemonsets
817 daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)" 850 daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)"
818 if [ "${daemonset_count}" != "12" ]; then 851 if [ "${daemonset_count}" != "13" ]; then
819 echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}" 852 echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}"
820 echo "Expected 12 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}" 853 echo "Expected 13 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
821 exit 1 854 exit 1
822 else 855 else
823 echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}" 856 echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}"
@@ -995,13 +1028,14 @@ init_default_state
995# run tests 1028# run tests
996install_base 1029install_base
997test_sysctl 1030test_sysctl
1031test_limits
998test_mounts 1032test_mounts
999test_ethtool 1033test_ethtool
1000test_uamlite 1034test_uamlite
1001purge_containers 1035purge_containers
1002test_overrides 1036test_overrides
1003 1037
1004# retore initial state 1038# restore initial state
1005init_default_state 1039init_default_state
1006 1040
1007echo "All tests pass for ${NAME}" 1041echo "All tests pass for ${NAME}"