diff --git a/divingbell/templates/bin/_apt.sh.tpl b/divingbell/templates/bin/_apt.sh.tpl index e34d8d4..a16ec42 100644 --- a/divingbell/templates/bin/_apt.sh.tpl +++ b/divingbell/templates/bin/_apt.sh.tpl @@ -85,10 +85,10 @@ rm -rf /etc/apt/sources.list.d/* mv /etc/apt/trusted.gpg.d/divindbell_temp.gpg /etc/apt/trusted.gpg.d/divindbell.gpg rm -f /etc/apt/trusted.gpg find /etc/apt/trusted.gpg.d/ -type f ! -name 'divindbell.gpg' -exec rm {{ "{}" }} \; -apt-get update +DEBIAN_FRONTEND=noninteractive apt-get update {{- end }} {{- if hasKey .Values.conf.apt "packages" }} -apt-get update +DEBIAN_FRONTEND=noninteractive apt-get update {{/* Build a unified list of packages */}} {{- $all_apt_packages := list }} @@ -114,6 +114,7 @@ apt-get update dpkg --configure -a # Perform package installs +set +x {{- range $all_apt_packages }} {{- $pkg_name := .name }} if [[ "${CURRENT_PACKAGES[{{ .name | squote }}]+isset}" != "isset"{{- if .version }} || "${CURRENT_PACKAGES[{{ .name | squote }}]}" != {{ .version }}{{- end }} ]]; then @@ -121,8 +122,9 @@ if [[ "${CURRENT_PACKAGES[{{ .name | squote }}]+isset}" != "isset"{{- if .versio fi REQUESTED_PACKAGES="$REQUESTED_PACKAGES {{$pkg_name}}" {{- end }} +set -x # Run this in case some package installation was interrupted -DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold {{- if .allow_downgrade }} "--allow-downgrades" {{ end }}{{- if .repo }} -t {{ .repo }}{{ end }} $INSTALLED_THIS_TIME +DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold {{- if .Values.conf.apt.allow_downgrade }} "--allow-downgrades" {{ end }}{{- if .repo }} -t {{ .repo }}{{ end }} $INSTALLED_THIS_TIME {{- end }} # Perform package upgrades @@ -141,17 +143,51 @@ fi #Remove packages not present in conf.apt anymore ################################################ +{{- if .Values.conf.apt.strict }} +APT_PURGE="apt-get purge -y --autoremove --allow-remove-essential" +{{- else }} +APT_PURGE="apt-get purge -y --autoremove" +{{- end }} + +{{- if hasKey .Values.conf.apt "packages" }} +{{- if .Values.conf.apt.strict }} +# in strict mode we execute this stage even on first run, so +# touch the packages file here to avoid the short-circuit below +touch ${persist_path}/packages +{{- end }} + echo $INSTALLED_THIS_TIME | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.new echo $REQUESTED_PACKAGES | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.requested if [ -f ${persist_path}/packages ]; then + # if strict mode, we reload the current package list to ensure we have an accurate list to audit from + # (e.g., in case a package was requested but not installed for some reason) + # note that in strict mode, $CURRENT_PACKAGES will duplicate the packages in $INSTALLED_THIS_TIME but in + # non-strict mode (which has logic to use the "packages" file it writes so it doesn't touch anything it + # didn't originally install) it doesn't. + {{- if .Values.conf.apt.strict }} + load_package_list_with_versions $(dpkg -l | awk 'NR>5 {print $2"="$3}') + {{- end }} + set +x + for package in "${!CURRENT_PACKAGES[@]}" + do + CURRENT_PACKAGE_NAMES="$CURRENT_PACKAGE_NAMES $package" + done + set -x + echo $CURRENT_PACKAGE_NAMES | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.current + {{- if .Values.conf.apt.strict }} + TO_DELETE=$(comm -23 ${persist_path}/packages.current ${persist_path}/packages.requested) + TO_KEEP=$(echo "$TO_DELETE" | comm -23 ${persist_path}/packages.current -) + {{- else }} TO_DELETE=$(comm -23 ${persist_path}/packages ${persist_path}/packages.requested) TO_KEEP=$(echo "$TO_DELETE" | comm -23 ${persist_path}/packages -) + {{- end }} if [ ! -z "$TO_DELETE" ]; then dpkg --configure -a - for pkg in "$TO_DELETE"; do - apt-get purge -y $pkg - done - apt-get autoremove -y + PURGE_LIST="" + while read -r pkg; do + PURGE_LIST="$PURGE_LIST $pkg" + done <<< "$TO_DELETE" + DEBIAN_FRONTEND=noninteractive $APT_PURGE $PURGE_LIST fi if [ ! -z "$TO_KEEP" ]; then echo "$TO_KEEP" > ${persist_path}/packages @@ -160,9 +196,12 @@ if [ -f ${persist_path}/packages ]; then fi fi if [ ! -z "$INSTALLED_THIS_TIME" ]; then +{{- if not .Values.conf.apt.strict }} cat ${persist_path}/packages.new >> ${persist_path}/packages +{{- end }} sort ${persist_path}/packages -o ${persist_path}/packages fi +{{- end }} ###################################################### #Stage 4 @@ -173,9 +212,8 @@ fi dpkg --configure -a {{- range .Values.conf.apt.blacklistpkgs }} {{- $package := . }} - apt-get remove --autoremove -y {{ $package | squote }} + DEBIAN_FRONTEND=noninteractive $APT_PURGE {{ $package | squote }} {{- end }} -apt-get autoremove -y {{- end }} log.INFO 'Putting the daemon to sleep.' diff --git a/divingbell/values.yaml b/divingbell/values.yaml index 1645e40..46aaac6 100644 --- a/divingbell/values.yaml +++ b/divingbell/values.yaml @@ -26,6 +26,8 @@ conf: log_colors: False apt: upgrade: false + allow_downgrade: false + strict: false blacklistpkgs: - telnetd - inetutils-telnetd diff --git a/doc/source/index.rst b/doc/source/index.rst index db2f200..5f0be70 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -113,15 +113,29 @@ want to remove from the configuration). When ``conf.apt.upgrade`` is ``true``, packages are upgraded `after` the requested packages are installed. +.. NOTE:: + + When ``conf.apt.allow_downgrade`` is ``true``, the ``--allow-downgrades`` + flag is passed to ``apt-get install``, allowing it to downgrade a package + if so specified in your packages list. + +.. NOTE:: + + When ``conf.apt.strict`` is ``true``, any packages not in conf.apt.packages + will be removed regardless of whether or not divingbell previously installed + them. (The default behavior is for only packages previously installed by + divingbell to be removed.) USE THIS OPTION WITH EXTREME CAUTION. + Here is an example configuration for it:: conf: apt: upgrade: false + allow_downgrade: false + strict: false packages: - name: version: - allow_downgrade: true - name: It is also permissible to use ``conf.apt.packages`` as a map, in which case all @@ -137,7 +151,6 @@ guarantees). For example:: group1: - name: version: - allow_downgrade: true - name: group2: - name: @@ -150,7 +163,6 @@ Is equivalent to:: packages: - name: version: - allow_downgrade: true - name: - name: - name: diff --git a/tools/gate/scripts/020-test-divingbell.sh b/tools/gate/scripts/020-test-divingbell.sh index 58506a8..d053b94 100755 --- a/tools/gate/scripts/020-test-divingbell.sh +++ b/tools/gate/scripts/020-test-divingbell.sh @@ -81,6 +81,16 @@ APT_PACKAGE5=python-setuptools APT_PACKAGE6=telnetd APT_PACKAGE7=sudoku APT_PACKAGE8=ninvaders +# helper function to generate a yaml config for all installed packages +APT_YAML_SEPARATOR=$'\n - name: ' +build_all_packages_yaml(){ + set +x + for f in "$@"; do + IFS=":" read -r name arch <<< $f; + APT_ALL_INSTALLED_PACKAGES="${APT_ALL_INSTALLED_PACKAGES}${APT_YAML_SEPARATOR}${name}" + done + set -x +} APT_REPOSITORY1="http://us.archive.ubuntu.com/ubuntu/" APT_DISTRIBUTIONS1="[ xenial ]" APT_COMPONENTS1="[ main, universe, restricted, multiverse ]" @@ -355,6 +365,8 @@ _reset_account(){ } init_default_state(){ + # TODO (dc6350) this needs retry logic to avoid race condition where tiller is not ready yet + sleep 30 # temporary fix for race condition purge_containers clean_persistent_files # set sysctl original vals @@ -1207,10 +1219,10 @@ test_apt(){ local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml echo "conf: apt: + allow_downgrade: true packages: - name: $APT_PACKAGE1 version: $APT_VERSION1 - allow_downgrade: true - name: $APT_PACKAGE2" > "${overrides_yaml}" install_base "--values=${overrides_yaml}" get_container_status apt @@ -1366,6 +1378,37 @@ $(printf '%s' "$APT_GPGKEY1" | awk '{printf " %s\n", $0}')" > "${overri _test_apt_package_version $APT_PACKAGE7 any _test_apt_package_version $APT_PACKAGE8 any echo '[SUCCESS] apt test9 passed successfully' >> "${TEST_RESULTS}" + + # Test adding a package in strict mode + local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set9.yaml + APT_ALL_INSTALLED_PACKAGES=" packages:" + build_all_packages_yaml $(dpkg -l | awk 'NR>5 {print $2}') + echo "conf: + apt: + strict: true +$APT_ALL_INSTALLED_PACKAGES + - name: $APT_PACKAGE1" > "${overrides_yaml}" + install_base "--values=${overrides_yaml}" + get_container_status apt + _test_apt_package_version $APT_PACKAGE1 any + # PACKAGE4 used earlier is intended to be a package that is always installed + _test_apt_package_version $APT_PACKAGE4 any + echo '[SUCCESS] apt test10 passed successfully' >> "${TEST_RESULTS}" + + # Test removing a package in strict mode + local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set10.yaml + # using the same APT_ALL_INSTALLED_PACKAGES from above, + # which does NOT have APT_PACKAGE1 + echo "conf: + apt: + strict: true +$APT_ALL_INSTALLED_PACKAGES" > "${overrides_yaml}" + install_base "--values=${overrides_yaml}" + get_container_status apt + _test_apt_package_version $APT_PACKAGE1 none + # PACKAGE4 used earlier is intended to be a package that is always installed + _test_apt_package_version $APT_PACKAGE4 any + echo '[SUCCESS] apt test11 passed successfully' >> "${TEST_RESULTS}" } # test exec module