From 44525162a59fb8a0c616ad00d2bda4b539e00423 Mon Sep 17 00:00:00 2001 From: "Crank, Daniel" Date: Wed, 15 Jan 2020 17:02:07 -0600 Subject: [PATCH] Add "strict" mode for apt package removal Currently, divingbell-apt will only remove packages that aren't on the current requested package list when they were previously installed by divingbell-apt. This patchset adds a "strict" mode which causes it to remove packages not on the requested package list regardless of whether divingbell installed them (i.e., it can remove unwanted packages that were part of the host's base image). Change-Id: Ie2ba5d47646bfaaf030cb54673e644ab0e917fd4 --- divingbell/templates/bin/_apt.sh.tpl | 56 +++++++++++++++++++---- divingbell/values.yaml | 2 + doc/source/index.rst | 18 ++++++-- tools/gate/scripts/020-test-divingbell.sh | 45 +++++++++++++++++- 4 files changed, 108 insertions(+), 13 deletions(-) diff --git a/divingbell/templates/bin/_apt.sh.tpl b/divingbell/templates/bin/_apt.sh.tpl index e34d8d4..a16ec42 100644 --- a/divingbell/templates/bin/_apt.sh.tpl +++ b/divingbell/templates/bin/_apt.sh.tpl @@ -85,10 +85,10 @@ rm -rf /etc/apt/sources.list.d/* mv /etc/apt/trusted.gpg.d/divindbell_temp.gpg /etc/apt/trusted.gpg.d/divindbell.gpg rm -f /etc/apt/trusted.gpg find /etc/apt/trusted.gpg.d/ -type f ! -name 'divindbell.gpg' -exec rm {{ "{}" }} \; -apt-get update +DEBIAN_FRONTEND=noninteractive apt-get update {{- end }} {{- if hasKey .Values.conf.apt "packages" }} -apt-get update +DEBIAN_FRONTEND=noninteractive apt-get update {{/* Build a unified list of packages */}} {{- $all_apt_packages := list }} @@ -114,6 +114,7 @@ apt-get update dpkg --configure -a # Perform package installs +set +x {{- range $all_apt_packages }} {{- $pkg_name := .name }} if [[ "${CURRENT_PACKAGES[{{ .name | squote }}]+isset}" != "isset"{{- if .version }} || "${CURRENT_PACKAGES[{{ .name | squote }}]}" != {{ .version }}{{- end }} ]]; then @@ -121,8 +122,9 @@ if [[ "${CURRENT_PACKAGES[{{ .name | squote }}]+isset}" != "isset"{{- if .versio fi REQUESTED_PACKAGES="$REQUESTED_PACKAGES {{$pkg_name}}" {{- end }} +set -x # Run this in case some package installation was interrupted -DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold {{- if .allow_downgrade }} "--allow-downgrades" {{ end }}{{- if .repo }} -t {{ .repo }}{{ end }} $INSTALLED_THIS_TIME +DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold {{- if .Values.conf.apt.allow_downgrade }} "--allow-downgrades" {{ end }}{{- if .repo }} -t {{ .repo }}{{ end }} $INSTALLED_THIS_TIME {{- end }} # Perform package upgrades @@ -141,17 +143,51 @@ fi #Remove packages not present in conf.apt anymore ################################################ +{{- if .Values.conf.apt.strict }} +APT_PURGE="apt-get purge -y --autoremove --allow-remove-essential" +{{- else }} +APT_PURGE="apt-get purge -y --autoremove" +{{- end }} + +{{- if hasKey .Values.conf.apt "packages" }} +{{- if .Values.conf.apt.strict }} +# in strict mode we execute this stage even on first run, so +# touch the packages file here to avoid the short-circuit below +touch ${persist_path}/packages +{{- end }} + echo $INSTALLED_THIS_TIME | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.new echo $REQUESTED_PACKAGES | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.requested if [ -f ${persist_path}/packages ]; then + # if strict mode, we reload the current package list to ensure we have an accurate list to audit from + # (e.g., in case a package was requested but not installed for some reason) + # note that in strict mode, $CURRENT_PACKAGES will duplicate the packages in $INSTALLED_THIS_TIME but in + # non-strict mode (which has logic to use the "packages" file it writes so it doesn't touch anything it + # didn't originally install) it doesn't. + {{- if .Values.conf.apt.strict }} + load_package_list_with_versions $(dpkg -l | awk 'NR>5 {print $2"="$3}') + {{- end }} + set +x + for package in "${!CURRENT_PACKAGES[@]}" + do + CURRENT_PACKAGE_NAMES="$CURRENT_PACKAGE_NAMES $package" + done + set -x + echo $CURRENT_PACKAGE_NAMES | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.current + {{- if .Values.conf.apt.strict }} + TO_DELETE=$(comm -23 ${persist_path}/packages.current ${persist_path}/packages.requested) + TO_KEEP=$(echo "$TO_DELETE" | comm -23 ${persist_path}/packages.current -) + {{- else }} TO_DELETE=$(comm -23 ${persist_path}/packages ${persist_path}/packages.requested) TO_KEEP=$(echo "$TO_DELETE" | comm -23 ${persist_path}/packages -) + {{- end }} if [ ! -z "$TO_DELETE" ]; then dpkg --configure -a - for pkg in "$TO_DELETE"; do - apt-get purge -y $pkg - done - apt-get autoremove -y + PURGE_LIST="" + while read -r pkg; do + PURGE_LIST="$PURGE_LIST $pkg" + done <<< "$TO_DELETE" + DEBIAN_FRONTEND=noninteractive $APT_PURGE $PURGE_LIST fi if [ ! -z "$TO_KEEP" ]; then echo "$TO_KEEP" > ${persist_path}/packages @@ -160,9 +196,12 @@ if [ -f ${persist_path}/packages ]; then fi fi if [ ! -z "$INSTALLED_THIS_TIME" ]; then +{{- if not .Values.conf.apt.strict }} cat ${persist_path}/packages.new >> ${persist_path}/packages +{{- end }} sort ${persist_path}/packages -o ${persist_path}/packages fi +{{- end }} ###################################################### #Stage 4 @@ -173,9 +212,8 @@ fi dpkg --configure -a {{- range .Values.conf.apt.blacklistpkgs }} {{- $package := . }} - apt-get remove --autoremove -y {{ $package | squote }} + DEBIAN_FRONTEND=noninteractive $APT_PURGE {{ $package | squote }} {{- end }} -apt-get autoremove -y {{- end }} log.INFO 'Putting the daemon to sleep.' diff --git a/divingbell/values.yaml b/divingbell/values.yaml index 1645e40..46aaac6 100644 --- a/divingbell/values.yaml +++ b/divingbell/values.yaml @@ -26,6 +26,8 @@ conf: log_colors: False apt: upgrade: false + allow_downgrade: false + strict: false blacklistpkgs: - telnetd - inetutils-telnetd diff --git a/doc/source/index.rst b/doc/source/index.rst index db2f200..5f0be70 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -113,15 +113,29 @@ want to remove from the configuration). When ``conf.apt.upgrade`` is ``true``, packages are upgraded `after` the requested packages are installed. +.. NOTE:: + + When ``conf.apt.allow_downgrade`` is ``true``, the ``--allow-downgrades`` + flag is passed to ``apt-get install``, allowing it to downgrade a package + if so specified in your packages list. + +.. NOTE:: + + When ``conf.apt.strict`` is ``true``, any packages not in conf.apt.packages + will be removed regardless of whether or not divingbell previously installed + them. (The default behavior is for only packages previously installed by + divingbell to be removed.) USE THIS OPTION WITH EXTREME CAUTION. + Here is an example configuration for it:: conf: apt: upgrade: false + allow_downgrade: false + strict: false packages: - name: version: - allow_downgrade: true - name: It is also permissible to use ``conf.apt.packages`` as a map, in which case all @@ -137,7 +151,6 @@ guarantees). For example:: group1: - name: version: - allow_downgrade: true - name: group2: - name: @@ -150,7 +163,6 @@ Is equivalent to:: packages: - name: version: - allow_downgrade: true - name: - name: - name: diff --git a/tools/gate/scripts/020-test-divingbell.sh b/tools/gate/scripts/020-test-divingbell.sh index 58506a8..d053b94 100755 --- a/tools/gate/scripts/020-test-divingbell.sh +++ b/tools/gate/scripts/020-test-divingbell.sh @@ -81,6 +81,16 @@ APT_PACKAGE5=python-setuptools APT_PACKAGE6=telnetd APT_PACKAGE7=sudoku APT_PACKAGE8=ninvaders +# helper function to generate a yaml config for all installed packages +APT_YAML_SEPARATOR=$'\n - name: ' +build_all_packages_yaml(){ + set +x + for f in "$@"; do + IFS=":" read -r name arch <<< $f; + APT_ALL_INSTALLED_PACKAGES="${APT_ALL_INSTALLED_PACKAGES}${APT_YAML_SEPARATOR}${name}" + done + set -x +} APT_REPOSITORY1="http://us.archive.ubuntu.com/ubuntu/" APT_DISTRIBUTIONS1="[ xenial ]" APT_COMPONENTS1="[ main, universe, restricted, multiverse ]" @@ -355,6 +365,8 @@ _reset_account(){ } init_default_state(){ + # TODO (dc6350) this needs retry logic to avoid race condition where tiller is not ready yet + sleep 30 # temporary fix for race condition purge_containers clean_persistent_files # set sysctl original vals @@ -1207,10 +1219,10 @@ test_apt(){ local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml echo "conf: apt: + allow_downgrade: true packages: - name: $APT_PACKAGE1 version: $APT_VERSION1 - allow_downgrade: true - name: $APT_PACKAGE2" > "${overrides_yaml}" install_base "--values=${overrides_yaml}" get_container_status apt @@ -1366,6 +1378,37 @@ $(printf '%s' "$APT_GPGKEY1" | awk '{printf " %s\n", $0}')" > "${overri _test_apt_package_version $APT_PACKAGE7 any _test_apt_package_version $APT_PACKAGE8 any echo '[SUCCESS] apt test9 passed successfully' >> "${TEST_RESULTS}" + + # Test adding a package in strict mode + local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set9.yaml + APT_ALL_INSTALLED_PACKAGES=" packages:" + build_all_packages_yaml $(dpkg -l | awk 'NR>5 {print $2}') + echo "conf: + apt: + strict: true +$APT_ALL_INSTALLED_PACKAGES + - name: $APT_PACKAGE1" > "${overrides_yaml}" + install_base "--values=${overrides_yaml}" + get_container_status apt + _test_apt_package_version $APT_PACKAGE1 any + # PACKAGE4 used earlier is intended to be a package that is always installed + _test_apt_package_version $APT_PACKAGE4 any + echo '[SUCCESS] apt test10 passed successfully' >> "${TEST_RESULTS}" + + # Test removing a package in strict mode + local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set10.yaml + # using the same APT_ALL_INSTALLED_PACKAGES from above, + # which does NOT have APT_PACKAGE1 + echo "conf: + apt: + strict: true +$APT_ALL_INSTALLED_PACKAGES" > "${overrides_yaml}" + install_base "--values=${overrides_yaml}" + get_container_status apt + _test_apt_package_version $APT_PACKAGE1 none + # PACKAGE4 used earlier is intended to be a package that is always installed + _test_apt_package_version $APT_PACKAGE4 any + echo '[SUCCESS] apt test11 passed successfully' >> "${TEST_RESULTS}" } # test exec module