Merge "Add the ability to install packages via divingbell"

This commit is contained in:
Zuul 2018-11-13 14:09:22 +00:00 committed by Gerrit Code Review
commit 96e2b073f3
6 changed files with 338 additions and 7 deletions

View File

@ -0,0 +1,112 @@
#!/bin/bash
{{/*
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/}}
set -e
cat <<'EOF' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apt.sh
{{ include "divingbell.shcommon" . }}
persist_path='/var/divingbell/apt'
declare -A CURRENT_PACKAGES
declare INSTALLED_THIS_TIME
declare TO_DELETE
declare TO_KEEP
declare REQUESTED_PACKAGES
if [ ! -d "${persist_path}" ]; then
mkdir -p "${persist_path}"
fi
write_test "${persist_path}"
load_package_list_with_versions(){
set +x
for f in "$@"; do
IFS="=" read -r name version <<< $f;
IFS=":" read -r name arch <<< $name;
CURRENT_PACKAGES["$name"]="$version";
done
set -x
}
################################################
#Stage 1
#Collect data
################################################
# First 5 lines are field descriptions
load_package_list_with_versions $(dpkg -l | awk 'NR>5 {print $2"="$3}')
################################################
#Stage 2
#Install new packages
################################################
{{- if hasKey .Values.conf "apt" }}
{{- if hasKey .Values.conf.apt "packages" }}
{{- range .Values.conf.apt.packages }}
if [[ "${CURRENT_PACKAGES[{{ .name | squote }}]+isset}" != "isset"{{- if .version }} || "${CURRENT_PACKAGES[{{ .name | squote }}]}" != {{ .version | squote }}{{- end }} ]]; then
apt-get install -y{{ if .repo }} -t {{ .repo | squote }}{{ end }} {{ .name | squote -}} {{- if .version }}={{ .version | squote }}{{ end }}
INSTALLED_THIS_TIME="$INSTALLED_THIS_TIME {{ .name }}"
fi
REQUESTED_PACKAGES="$REQUESTED_PACKAGES {{ .name }}"
{{- end }}
{{- end }}
{{- end }}
################################################
#Stage 3
#Remove packages not present in conf.apt anymore
################################################
echo $INSTALLED_THIS_TIME | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.new
echo $REQUESTED_PACKAGES | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.requested
if [ -f ${persist_path}/packages ]; then
TO_DELETE=$(comm -23 ${persist_path}/packages ${persist_path}/packages.requested)
TO_KEEP=$(echo "$TO_DELETE" | comm -23 ${persist_path}/packages -)
if [ ! -z "$TO_DELETE" ]; then
for pkg in "$TO_DELETE"; do
apt-get purge -y $pkg
done
apt-get autoremove -y
fi
if [ ! -z "$TO_KEEP" ]; then
echo "$TO_KEEP" > ${persist_path}/packages
else
rm ${persist_path}/packages
fi
fi
if [ ! -z "$INSTALLED_THIS_TIME" ]; then
cat ${persist_path}/packages.new >> ${persist_path}/packages
sort ${persist_path}/packages -o ${persist_path}/packages
fi
exit 0
EOF
chmod 755 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apt.sh
chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/apt.sh
sleep 1
echo 'INFO Putting the daemon to sleep.'
while [ 1 ]; do
sleep 300
done
exit 0

View File

@ -0,0 +1,69 @@
{{/*
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/}}
{{- define "divingbell.daemonset.apt" }}
{{- $daemonset := index . 0 }}
{{- $secretName := index . 1 }}
{{- $envAll := index . 2 }}
{{- with $envAll }}
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: {{ $daemonset }}
spec:
{{ tuple $envAll $daemonset | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
template:
metadata:
labels:
{{ list $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
hostNetwork: true
hostPID: true
hostIPC: true
containers:
- name: {{ $daemonset }}
image: {{ .Values.images.divingbell }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.apt | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
command:
- /tmp/{{ $daemonset }}.sh
volumeMounts:
- name: rootfs-{{ $daemonset }}
mountPath: {{ .Values.conf.chroot_mnt_path }}
- name: {{ $secretName }}
mountPath: /tmp/{{ $daemonset }}.sh
subPath: {{ $daemonset }}
readOnly: true
securityContext:
privileged: true
volumes:
- name: rootfs-{{ $daemonset }}
hostPath:
path: /
- name: {{ $secretName }}
secret:
secretName: {{ $secretName }}
defaultMode: 0555
{{- end }}
{{- end }}
{{- if .Values.manifests.daemonset_apt }}
{{- $daemonset := "apt" }}
{{- $secretName := "divingbell-apt" }}
{{- $daemonset_yaml := list $daemonset $secretName . | include "divingbell.daemonset.apt" | toString | fromYaml }}
{{- $secret_include := "divingbell.secret.apt" }}
{{- list $daemonset $daemonset_yaml $secret_include $secretName . | include "helm-toolkit.utils.daemonset_overrides" }}
{{- end }}

View File

@ -0,0 +1,30 @@
{{/*
Copyright 2018 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "divingbell.secret.apt" }}
{{- $secretName := index . 0 }}
{{- $envAll := index . 1 }}
{{- with $envAll }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
data:
apt: |+
{{ tuple "bin/_apt.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc | indent 4 }}
{{- end }}
{{- end }}

View File

@ -61,6 +61,10 @@ pod:
enabled: true
min_ready_seconds: 0
max_unavailable: 100%
apt:
enabled: true
min_ready_seconds: 0
max_unavailable: 100%
limits:
enabled: true
min_ready_seconds: 0
@ -102,6 +106,13 @@ pod:
requests:
memory: "128Mi"
cpu: "100m"
apt:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
manifests:
daemonset_ethtool: true
@ -109,3 +120,4 @@ manifests:
daemonset_uamlite: true
daemonset_sysctl: true
daemonset_limits: true
daemonset_apt: true

View File

@ -124,11 +124,6 @@ Used to manage host level NIC tunables. Ex::
tx-tcp-segmentation: off
tx-checksum-ip-generic: on
packages
^^^^^^^^
Not implemented
uamlite
^^^^^^^
@ -146,6 +141,23 @@ access. Ex::
- ssh-rsa AAAAB3N... key1-comment
- ssh-rsa AAAAVY6... key2-comment
apt
^^^
``apt`` daemonset does package management. It is able to install a package of
a specific version (or upgrade an existing one to requested version). Version
is optional, and if not provided the latest available package is installed.
It can also remove packages that were previously installed by divingbell (it is
done by excluding the packages you want to remove from the configuration).
Here is an example configuration for it::
conf:
apt:
packages:
- name: <PACKAGE1>
version: <VERSION1>
- name: <PACKAGE2>
Operations
----------

View File

@ -47,6 +47,13 @@ USERNAME3=userthree
USERNAME3_SUDO=true
USERNAME4=userfour
USERNAME4_SUDO=false
APT_PACKAGE1=python-pbr
APT_VERSION1=1.8.0-4ubuntu1
APT_PACKAGE2=python-yaml
APT_PACKAGE3=python-simplejson
APT_VERSION3=3.8.1-1ubuntu2
APT_PACKAGE4=less
APT_PACKAGE5=python-setuptools
type lshw || apt -y install lshw
nic_info="$(lshw -class network)"
physical_nic=''
@ -753,6 +760,94 @@ test_uamlite(){
echo '[SUCCESS] uamlite test6 passed successfully' >> "${TEST_RESULTS}"
}
_test_apt_package_version(){
local pkg_name=$1
local pkg_ver=$2
if [ ${pkg_ver} = "none" ]; then
if [[ $(dpkg -l | grep ${pkg_name}) ]]; then
echo "[FAIL] Package ${pkg_name} should not be installed" >> "${TEST_RESULTS}"
return 1
fi
elif [ ${pkg_ver} = "any" ]; then
if [[ ! $(dpkg -l | grep ${pkg_name}) ]]; then
echo "[FAIL] Package ${pkg_name} should be installed" >> "${TEST_RESULTS}"
return 1
fi
else
if [ $(dpkg -l | awk "/[[:space:]]${pkg_name}[[:space:]]/"'{print $3}') != "${pkg_ver}" ]; then
echo "[FAIL] Package ${pkg_name} should be of version ${pkg_ver}" >> "${TEST_RESULTS}"
return 1
fi
fi
}
test_apt(){
# Test the valid set of packages
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
echo "conf:
apt:
packages:
- name: $APT_PACKAGE1
version: $APT_VERSION1
- name: $APT_PACKAGE2" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt
_test_apt_package_version $APT_PACKAGE1 $APT_VERSION1
_test_apt_package_version $APT_PACKAGE2 any
echo '[SUCCESS] apt test1 passed successfully' >> "${TEST_RESULTS}"
# Test removal of one package and install of one new package
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
echo "conf:
apt:
packages:
- name: $APT_PACKAGE2
- name: $APT_PACKAGE3
version: $APT_VERSION3" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt
_test_apt_package_version $APT_PACKAGE1 none
_test_apt_package_version $APT_PACKAGE2 any
_test_apt_package_version $APT_PACKAGE3 $APT_VERSION3
echo '[SUCCESS] apt test2 passed successfully' >> "${TEST_RESULTS}"
# Test removal of all installed packages and install of one that already exists
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set3.yaml
echo "conf:
apt:
packages:
- name: $APT_PACKAGE4" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt
_test_apt_package_version $APT_PACKAGE2 none
_test_apt_package_version $APT_PACKAGE3 none
echo '[SUCCESS] apt test3 passed successfully' >> "${TEST_RESULTS}"
# Test package not installed by divingbell not removed
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set4.yaml
echo "conf:
apt:
packages:
- name: $APT_PACKAGE5" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt
_test_apt_package_version $APT_PACKAGE4 any # Should still be present
_test_apt_package_version $APT_PACKAGE5 any
echo '[SUCCESS] apt test4 passed successfully' >> "${TEST_RESULTS}"
# Test invalid package name
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
echo "conf:
apt:
packages:
- name: some-random-name
version: whatever" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apt expect_failure
_test_clog_msg 'E: Unable to locate package some-random-name'
echo '[SUCCESS] apt test5 passed successfully' >> "${TEST_RESULTS}"
}
# test daemonset value overrides for hosts and labels
test_overrides(){
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-dryrun.yaml
@ -848,9 +943,9 @@ test_overrides(){
# Compare against expected number of generated daemonsets
daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)"
if [ "${daemonset_count}" != "13" ]; then
if [ "${daemonset_count}" != "14" ]; then
echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}"
echo "Expected 13 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
echo "Expected 14 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
exit 1
else
echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}"
@ -1032,6 +1127,7 @@ test_limits
test_mounts
test_ethtool
test_uamlite
test_apt
purge_containers
test_overrides