summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVladyslav Drok <vdrok@mirantis.com>2018-10-22 17:21:00 -0700
committerVladyslav Drok <vdrok@mirantis.com>2018-11-12 13:34:58 -0800
commit0f5567dc587ee0991ecc5628abdd52df5dd4c67e (patch)
tree9de0e7138d206be05fc66cd3603642e310afec05
parent461f4e93d5e56c2327005aa13f7660b377704a4d (diff)
Add the ability to install packages via divingbell
This change adds a possibility to install or upgrade to packages with a specific version. The daemonset also tracks the packages installed, and will be removing the packages that were deleted from the chart but were previously installed by divingbell. Change-Id: Ia6066679e549190054eb2cf71589065177447447
Notes
Notes (review): Code-Review+2: Aaron Sheffield <ajs@sheffieldfamily.net> Code-Review+2: Scott Hussey <sthussey@att.com> Workflow+1: Scott Hussey <sthussey@att.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Tue, 13 Nov 2018 14:09:22 +0000 Reviewed-on: https://review.openstack.org/612540 Project: openstack/airship-divingbell Branch: refs/heads/master
-rw-r--r--divingbell/templates/bin/_apt.sh.tpl112
-rw-r--r--divingbell/templates/daemonset-apt.yaml69
-rw-r--r--divingbell/templates/secret-apt.yaml30
-rw-r--r--divingbell/values.yaml12
-rw-r--r--doc/source/index.rst22
-rwxr-xr-xtools/gate/scripts/020-test-divingbell.sh100
6 files changed, 338 insertions, 7 deletions
diff --git a/divingbell/templates/bin/_apt.sh.tpl b/divingbell/templates/bin/_apt.sh.tpl
new file mode 100644
index 0000000..1112981
--- /dev/null
+++ b/divingbell/templates/bin/_apt.sh.tpl
@@ -0,0 +1,112 @@
1#!/bin/bash
2
3{{/*
4# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17*/}}
18
19set -e
20
21cat <<'EOF' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apt.sh
22{{ include "divingbell.shcommon" . }}
23
24persist_path='/var/divingbell/apt'
25declare -A CURRENT_PACKAGES
26declare INSTALLED_THIS_TIME
27declare TO_DELETE
28declare TO_KEEP
29declare REQUESTED_PACKAGES
30
31if [ ! -d "${persist_path}" ]; then
32 mkdir -p "${persist_path}"
33fi
34
35write_test "${persist_path}"
36
37load_package_list_with_versions(){
38 set +x
39 for f in "$@"; do
40 IFS="=" read -r name version <<< $f;
41 IFS=":" read -r name arch <<< $name;
42 CURRENT_PACKAGES["$name"]="$version";
43 done
44 set -x
45}
46
47################################################
48#Stage 1
49#Collect data
50################################################
51
52# First 5 lines are field descriptions
53load_package_list_with_versions $(dpkg -l | awk 'NR>5 {print $2"="$3}')
54
55################################################
56#Stage 2
57#Install new packages
58################################################
59
60{{- if hasKey .Values.conf "apt" }}
61{{- if hasKey .Values.conf.apt "packages" }}
62{{- range .Values.conf.apt.packages }}
63if [[ "${CURRENT_PACKAGES[{{ .name | squote }}]+isset}" != "isset"{{- if .version }} || "${CURRENT_PACKAGES[{{ .name | squote }}]}" != {{ .version | squote }}{{- end }} ]]; then
64 apt-get install -y{{ if .repo }} -t {{ .repo | squote }}{{ end }} {{ .name | squote -}} {{- if .version }}={{ .version | squote }}{{ end }}
65 INSTALLED_THIS_TIME="$INSTALLED_THIS_TIME {{ .name }}"
66fi
67REQUESTED_PACKAGES="$REQUESTED_PACKAGES {{ .name }}"
68{{- end }}
69{{- end }}
70{{- end }}
71
72################################################
73#Stage 3
74#Remove packages not present in conf.apt anymore
75################################################
76
77echo $INSTALLED_THIS_TIME | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.new
78echo $REQUESTED_PACKAGES | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.requested
79if [ -f ${persist_path}/packages ]; then
80 TO_DELETE=$(comm -23 ${persist_path}/packages ${persist_path}/packages.requested)
81 TO_KEEP=$(echo "$TO_DELETE" | comm -23 ${persist_path}/packages -)
82 if [ ! -z "$TO_DELETE" ]; then
83 for pkg in "$TO_DELETE"; do
84 apt-get purge -y $pkg
85 done
86 apt-get autoremove -y
87 fi
88 if [ ! -z "$TO_KEEP" ]; then
89 echo "$TO_KEEP" > ${persist_path}/packages
90 else
91 rm ${persist_path}/packages
92 fi
93fi
94if [ ! -z "$INSTALLED_THIS_TIME" ]; then
95 cat ${persist_path}/packages.new >> ${persist_path}/packages
96 sort ${persist_path}/packages -o ${persist_path}/packages
97fi
98
99exit 0
100EOF
101
102chmod 755 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apt.sh
103chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/apt.sh
104
105sleep 1
106echo 'INFO Putting the daemon to sleep.'
107
108while [ 1 ]; do
109 sleep 300
110done
111
112exit 0
diff --git a/divingbell/templates/daemonset-apt.yaml b/divingbell/templates/daemonset-apt.yaml
new file mode 100644
index 0000000..eeb929b
--- /dev/null
+++ b/divingbell/templates/daemonset-apt.yaml
@@ -0,0 +1,69 @@
1{{/*
2# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15*/}}
16
17{{- define "divingbell.daemonset.apt" }}
18 {{- $daemonset := index . 0 }}
19 {{- $secretName := index . 1 }}
20 {{- $envAll := index . 2 }}
21 {{- with $envAll }}
22---
23apiVersion: extensions/v1beta1
24kind: DaemonSet
25metadata:
26 name: {{ $daemonset }}
27spec:
28{{ tuple $envAll $daemonset | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
29 template:
30 metadata:
31 labels:
32{{ list $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
33 spec:
34 hostNetwork: true
35 hostPID: true
36 hostIPC: true
37 containers:
38 - name: {{ $daemonset }}
39 image: {{ .Values.images.divingbell }}
40 imagePullPolicy: {{ .Values.images.pull_policy }}
41{{ tuple $envAll $envAll.Values.pod.resources.apt | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
42 command:
43 - /tmp/{{ $daemonset }}.sh
44 volumeMounts:
45 - name: rootfs-{{ $daemonset }}
46 mountPath: {{ .Values.conf.chroot_mnt_path }}
47 - name: {{ $secretName }}
48 mountPath: /tmp/{{ $daemonset }}.sh
49 subPath: {{ $daemonset }}
50 readOnly: true
51 securityContext:
52 privileged: true
53 volumes:
54 - name: rootfs-{{ $daemonset }}
55 hostPath:
56 path: /
57 - name: {{ $secretName }}
58 secret:
59 secretName: {{ $secretName }}
60 defaultMode: 0555
61 {{- end }}
62{{- end }}
63{{- if .Values.manifests.daemonset_apt }}
64{{- $daemonset := "apt" }}
65{{- $secretName := "divingbell-apt" }}
66{{- $daemonset_yaml := list $daemonset $secretName . | include "divingbell.daemonset.apt" | toString | fromYaml }}
67{{- $secret_include := "divingbell.secret.apt" }}
68{{- list $daemonset $daemonset_yaml $secret_include $secretName . | include "helm-toolkit.utils.daemonset_overrides" }}
69{{- end }}
diff --git a/divingbell/templates/secret-apt.yaml b/divingbell/templates/secret-apt.yaml
new file mode 100644
index 0000000..14f0a6d
--- /dev/null
+++ b/divingbell/templates/secret-apt.yaml
@@ -0,0 +1,30 @@
1{{/*
2Copyright 2018 The Openstack-Helm Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/}}
16
17{{- define "divingbell.secret.apt" }}
18{{- $secretName := index . 0 }}
19{{- $envAll := index . 1 }}
20{{- with $envAll }}
21---
22apiVersion: v1
23kind: Secret
24metadata:
25 name: {{ $secretName }}
26data:
27 apt: |+
28{{ tuple "bin/_apt.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc | indent 4 }}
29{{- end }}
30{{- end }}
diff --git a/divingbell/values.yaml b/divingbell/values.yaml
index 8436524..02a3594 100644
--- a/divingbell/values.yaml
+++ b/divingbell/values.yaml
@@ -61,6 +61,10 @@ pod:
61 enabled: true 61 enabled: true
62 min_ready_seconds: 0 62 min_ready_seconds: 0
63 max_unavailable: 100% 63 max_unavailable: 100%
64 apt:
65 enabled: true
66 min_ready_seconds: 0
67 max_unavailable: 100%
64 limits: 68 limits:
65 enabled: true 69 enabled: true
66 min_ready_seconds: 0 70 min_ready_seconds: 0
@@ -102,6 +106,13 @@ pod:
102 requests: 106 requests:
103 memory: "128Mi" 107 memory: "128Mi"
104 cpu: "100m" 108 cpu: "100m"
109 apt:
110 limits:
111 memory: "128Mi"
112 cpu: "100m"
113 requests:
114 memory: "128Mi"
115 cpu: "100m"
105 116
106manifests: 117manifests:
107 daemonset_ethtool: true 118 daemonset_ethtool: true
@@ -109,3 +120,4 @@ manifests:
109 daemonset_uamlite: true 120 daemonset_uamlite: true
110 daemonset_sysctl: true 121 daemonset_sysctl: true
111 daemonset_limits: true 122 daemonset_limits: true
123 daemonset_apt: true
diff --git a/doc/source/index.rst b/doc/source/index.rst
index df01788..2546e72 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -124,11 +124,6 @@ Used to manage host level NIC tunables. Ex::
124 tx-tcp-segmentation: off 124 tx-tcp-segmentation: off
125 tx-checksum-ip-generic: on 125 tx-checksum-ip-generic: on
126 126
127packages
128^^^^^^^^
129
130Not implemented
131
132uamlite 127uamlite
133^^^^^^^ 128^^^^^^^
134 129
@@ -146,6 +141,23 @@ access. Ex::
146 - ssh-rsa AAAAB3N... key1-comment 141 - ssh-rsa AAAAB3N... key1-comment
147 - ssh-rsa AAAAVY6... key2-comment 142 - ssh-rsa AAAAVY6... key2-comment
148 143
144apt
145^^^
146
147``apt`` daemonset does package management. It is able to install a package of
148a specific version (or upgrade an existing one to requested version). Version
149is optional, and if not provided the latest available package is installed.
150It can also remove packages that were previously installed by divingbell (it is
151done by excluding the packages you want to remove from the configuration).
152Here is an example configuration for it::
153
154 conf:
155 apt:
156 packages:
157 - name: <PACKAGE1>
158 version: <VERSION1>
159 - name: <PACKAGE2>
160
149Operations 161Operations
150---------- 162----------
151 163
diff --git a/tools/gate/scripts/020-test-divingbell.sh b/tools/gate/scripts/020-test-divingbell.sh
index 9042ec6..2809f43 100755
--- a/tools/gate/scripts/020-test-divingbell.sh
+++ b/tools/gate/scripts/020-test-divingbell.sh
@@ -47,6 +47,13 @@ USERNAME3=userthree
47USERNAME3_SUDO=true 47USERNAME3_SUDO=true
48USERNAME4=userfour 48USERNAME4=userfour
49USERNAME4_SUDO=false 49USERNAME4_SUDO=false
50APT_PACKAGE1=python-pbr
51APT_VERSION1=1.8.0-4ubuntu1
52APT_PACKAGE2=python-yaml
53APT_PACKAGE3=python-simplejson
54APT_VERSION3=3.8.1-1ubuntu2
55APT_PACKAGE4=less
56APT_PACKAGE5=python-setuptools
50type lshw || apt -y install lshw 57type lshw || apt -y install lshw
51nic_info="$(lshw -class network)" 58nic_info="$(lshw -class network)"
52physical_nic='' 59physical_nic=''
@@ -753,6 +760,94 @@ test_uamlite(){
753 echo '[SUCCESS] uamlite test6 passed successfully' >> "${TEST_RESULTS}" 760 echo '[SUCCESS] uamlite test6 passed successfully' >> "${TEST_RESULTS}"
754} 761}
755 762
763_test_apt_package_version(){
764 local pkg_name=$1
765 local pkg_ver=$2
766 if [ ${pkg_ver} = "none" ]; then
767 if [[ $(dpkg -l | grep ${pkg_name}) ]]; then
768 echo "[FAIL] Package ${pkg_name} should not be installed" >> "${TEST_RESULTS}"
769 return 1
770 fi
771 elif [ ${pkg_ver} = "any" ]; then
772 if [[ ! $(dpkg -l | grep ${pkg_name}) ]]; then
773 echo "[FAIL] Package ${pkg_name} should be installed" >> "${TEST_RESULTS}"
774 return 1
775 fi
776 else
777 if [ $(dpkg -l | awk "/[[:space:]]${pkg_name}[[:space:]]/"'{print $3}') != "${pkg_ver}" ]; then
778 echo "[FAIL] Package ${pkg_name} should be of version ${pkg_ver}" >> "${TEST_RESULTS}"
779 return 1
780 fi
781 fi
782}
783
784test_apt(){
785 # Test the valid set of packages
786 local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
787 echo "conf:
788 apt:
789 packages:
790 - name: $APT_PACKAGE1
791 version: $APT_VERSION1
792 - name: $APT_PACKAGE2" > "${overrides_yaml}"
793 install_base "--values=${overrides_yaml}"
794 get_container_status apt
795 _test_apt_package_version $APT_PACKAGE1 $APT_VERSION1
796 _test_apt_package_version $APT_PACKAGE2 any
797 echo '[SUCCESS] apt test1 passed successfully' >> "${TEST_RESULTS}"
798
799 # Test removal of one package and install of one new package
800 local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
801 echo "conf:
802 apt:
803 packages:
804 - name: $APT_PACKAGE2
805 - name: $APT_PACKAGE3
806 version: $APT_VERSION3" > "${overrides_yaml}"
807 install_base "--values=${overrides_yaml}"
808 get_container_status apt
809 _test_apt_package_version $APT_PACKAGE1 none
810 _test_apt_package_version $APT_PACKAGE2 any
811 _test_apt_package_version $APT_PACKAGE3 $APT_VERSION3
812 echo '[SUCCESS] apt test2 passed successfully' >> "${TEST_RESULTS}"
813
814 # Test removal of all installed packages and install of one that already exists
815 local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set3.yaml
816 echo "conf:
817 apt:
818 packages:
819 - name: $APT_PACKAGE4" > "${overrides_yaml}"
820 install_base "--values=${overrides_yaml}"
821 get_container_status apt
822 _test_apt_package_version $APT_PACKAGE2 none
823 _test_apt_package_version $APT_PACKAGE3 none
824 echo '[SUCCESS] apt test3 passed successfully' >> "${TEST_RESULTS}"
825
826 # Test package not installed by divingbell not removed
827 local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set4.yaml
828 echo "conf:
829 apt:
830 packages:
831 - name: $APT_PACKAGE5" > "${overrides_yaml}"
832 install_base "--values=${overrides_yaml}"
833 get_container_status apt
834 _test_apt_package_version $APT_PACKAGE4 any # Should still be present
835 _test_apt_package_version $APT_PACKAGE5 any
836 echo '[SUCCESS] apt test4 passed successfully' >> "${TEST_RESULTS}"
837
838 # Test invalid package name
839 overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
840 echo "conf:
841 apt:
842 packages:
843 - name: some-random-name
844 version: whatever" > "${overrides_yaml}"
845 install_base "--values=${overrides_yaml}"
846 get_container_status apt expect_failure
847 _test_clog_msg 'E: Unable to locate package some-random-name'
848 echo '[SUCCESS] apt test5 passed successfully' >> "${TEST_RESULTS}"
849}
850
756# test daemonset value overrides for hosts and labels 851# test daemonset value overrides for hosts and labels
757test_overrides(){ 852test_overrides(){
758 overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-dryrun.yaml 853 overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-dryrun.yaml
@@ -848,9 +943,9 @@ test_overrides(){
848 943
849 # Compare against expected number of generated daemonsets 944 # Compare against expected number of generated daemonsets
850 daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)" 945 daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)"
851 if [ "${daemonset_count}" != "13" ]; then 946 if [ "${daemonset_count}" != "14" ]; then
852 echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}" 947 echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}"
853 echo "Expected 13 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}" 948 echo "Expected 14 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
854 exit 1 949 exit 1
855 else 950 else
856 echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}" 951 echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}"
@@ -1032,6 +1127,7 @@ test_limits
1032test_mounts 1127test_mounts
1033test_ethtool 1128test_ethtool
1034test_uamlite 1129test_uamlite
1130test_apt
1035purge_containers 1131purge_containers
1036test_overrides 1132test_overrides
1037 1133