diff --git a/divingbell/templates/bin/_apparmor.sh.tpl b/divingbell/templates/bin/_apparmor.sh.tpl new file mode 100644 index 0000000..768e9bd --- /dev/null +++ b/divingbell/templates/bin/_apparmor.sh.tpl @@ -0,0 +1,137 @@ +#!/bin/bash + +{{/* +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +set -e + +cat <<'EOF' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apparmor_host.sh +{{ include "divingbell.shcommon" . }} + +load_flags="-r -W" +{{- if hasKey .Values.conf "apparmor" }} +{{- if hasKey .Values.conf.apparmor "complain_mode" }} +{{- if .Values.conf.apparmor.complain_mode }} +load_flags="$load_flags -C" +{{- end }} +{{- end }} +{{- end }} +load_cmd="apparmor_parser $load_flags" +unload_cmd='apparmor_parser -R' +defaults_path='/var/divingbell/apparmor' +persist_path='/etc/apparmor.d' +declare -A CURRENT_FILENAMES +declare -A SAVED_STATE_FILENAMES + +if [ ! -d "${defaults_path}" ]; then + mkdir -p "${defaults_path}" +fi + +write_test "${defaults_path}" +write_test "${persist_path}" + +save_apparmor_profile(){ + local filename="$1" + local data="$2" + CURRENT_FILENAMES["$filename"]='' + + #Check if host already had the same filename + if [ ${SAVED_STATE_FILENAMES["$filename"]+_} ]; then + unset SAVED_STATE_FILENAMES["$filename"] + fi + + echo -ne "${data}" > ${defaults_path}/${filename} + if [ ! -L ${persist_path}/${filename} ]; then + ln -s ${defaults_path}/${filename} ${persist_path}/${filename} + fi +} + +####################################### +#Stage 1 +#Collect data +####################################### + +#Search for any saved apparmor profiles +pushd $defaults_path +count=$(find . -type f | wc -l) + +#Check if directory is non-empty +if [ $count -gt 0 ]; then + for f in $(find . -type f|xargs -n1 basename); do + SAVED_STATE_FILENAMES[$f]='' + done +fi + +####################################### +#Stage 2 +#Save new apparmor profiles +####################################### + +{{- if hasKey .Values.conf "apparmor" }} +{{- if hasKey .Values.conf.apparmor "profiles" }} +{{- range $filename, $value := .Values.conf.apparmor.profiles }} +save_apparmor_profile {{ $filename | squote }} {{ $value | squote }} +{{- end }} +{{- end }} +{{- end }} + + +####################################### +#Stage 3 +#Clean stale apparmor profiles +####################################### + +#If hash is not empty - there are old filenames that need to be handled +if [ ${#SAVED_STATE_FILENAMES[@]} -gt 0 ]; then + for filename in ${!SAVED_STATE_FILENAMES[@]}; do + #Unload any previously applied apparmor profiles which are now absent + $unload_cmd ${defaults_path}/${filename} || die "Problem unloading profile ${defaults_path}/${filename}" + if [ -L ${persist_path}/${filename} ]; then + unlink ${persist_path}/${filename} + fi + rm -f ${defaults_path}/${filename} + # log/append the stale profiles that require eventual reboot + echo "apparmor: stale profile ${defaults_path}/${filename}" >> /var/run/reboot-required.pkgs + unset SAVED_STATE_FILENAMES["$filename"] + done + # mark node as needing eventual reboot + echo '*** System restart required ***' > /var/run/reboot-required +fi + +####################################### +#Stage 4 +#Install/update new apparmor profiles +#Save new apparmor profiles +####################################### + +for filename in ${!CURRENT_FILENAMES[@]}; do + $load_cmd ${persist_path}/${filename} || die "Problem loading ${persist_path}/${filename}" +done + +exit 0 +EOF + +chmod 755 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apparmor_host.sh +chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/apparmor_host.sh + +sleep 1 +echo 'INFO Putting the daemon to sleep.' + +while [ 1 ]; do + sleep 300 +done + +exit 0 diff --git a/divingbell/templates/daemonset-apparmor.yaml b/divingbell/templates/daemonset-apparmor.yaml new file mode 100644 index 0000000..6d673b0 --- /dev/null +++ b/divingbell/templates/daemonset-apparmor.yaml @@ -0,0 +1,69 @@ +{{/* +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +{{- define "divingbell.daemonset.apparmor" }} + {{- $daemonset := index . 0 }} + {{- $secretName := index . 1 }} + {{- $envAll := index . 2 }} + {{- with $envAll }} +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: {{ $daemonset }} +spec: +{{ tuple $envAll $daemonset | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ list $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + hostNetwork: true + hostPID: true + hostIPC: true + containers: + - name: {{ $daemonset }} + image: {{ .Values.images.divingbell }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.apparmor | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }} + command: + - /tmp/{{ $daemonset }}.sh + volumeMounts: + - name: rootfs-{{ $daemonset }} + mountPath: {{ .Values.conf.chroot_mnt_path }} + - name: {{ $secretName }} + mountPath: /tmp/{{ $daemonset }}.sh + subPath: {{ $daemonset }} + readOnly: true + securityContext: + privileged: true + volumes: + - name: rootfs-{{ $daemonset }} + hostPath: + path: / + - name: {{ $secretName }} + secret: + secretName: {{ $secretName }} + defaultMode: 0555 + {{- end }} +{{- end }} +{{- if .Values.manifests.daemonset_apparmor }} +{{- $daemonset := "apparmor" }} +{{- $secretName := "divingbell-apparmor" }} +{{- $daemonset_yaml := list $daemonset $secretName . | include "divingbell.daemonset.apparmor" | toString | fromYaml }} +{{- $secret_include := "divingbell.secret.apparmor" }} +{{- list $daemonset $daemonset_yaml $secret_include $secretName . | include "helm-toolkit.utils.daemonset_overrides" }} +{{- end }} diff --git a/divingbell/templates/secret-apparmor.yaml b/divingbell/templates/secret-apparmor.yaml new file mode 100644 index 0000000..28eebe6 --- /dev/null +++ b/divingbell/templates/secret-apparmor.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "divingbell.secret.apparmor" }} +{{- $secretName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +data: + apparmor: {{ tuple "bin/_apparmor.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} +{{- end }} diff --git a/divingbell/values.yaml b/divingbell/values.yaml index 5d3fc55..84fb1e9 100644 --- a/divingbell/values.yaml +++ b/divingbell/values.yaml @@ -127,6 +127,13 @@ pod: max_unavailable: 100% resources: enabled: false + apparmor: + limits: + memory: "128Mi" + cpu: "100m" + requests: + memory: "128Mi" + cpu: "100m" ethtool: limits: memory: "128Mi" @@ -193,3 +200,4 @@ manifests: daemonset_apt: true daemonset_perm: true daemonset_exec: true + daemonset_apparmor: true diff --git a/doc/source/index.rst b/doc/source/index.rst index ec74a98..fd7b677 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -250,9 +250,85 @@ The following set of options are not yet implemeneted:: failing script should be retried. Failed exec count does not persist through pod/node restart. Default value is ``infinite``. +apparmor +^^^^^^^^ + +Used to manage host level apparmor profiles/rules, Ex:: + + conf: + apparmor: + complain_mode: "true" + profiles: + profile-1: | + #include + /usr/sbin/profile-1 { + #include + #include + #include + + capability dac_override, + capability dac_read_search, + capability net_bind_service, + capability setgid, + capability setuid, + + /data/www/safe/* r, + deny /data/www/unsafe/* r, + } + profile-2: | + #include + /usr/sbin/profile-2 { + #include + #include + #include + + capability dac_override, + capability dac_read_search, + capability net_bind_service, + capability setgid, + capability setuid, + + /data/www/safe/* r, + deny /data/www/unsafe/* r, + } + Operations ---------- +Setting apparmor profiles +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The way apparmor loading/unloading implemented is through saving +settings to a file and than running ``apparmor_parser`` command. +The daemonset supports both enforcement and complain mode, +enforcement being the default. To request complain mode for the +profiles, add ``complain_mode: "true"`` nested under apparmor entry. + +It's easy to mess up host with rules, if profile names would +distinguish from file content. Ex:: + + conf: + apparmor: + profiles: + profile-1: | + #include + /usr/sbin/profile-1 { + #include + capability setgid, + } + profile-2: | + #include + /usr/sbin/profile-1 { + #include + capability net_bind_service, + } + +Even when profiles are different (profile-1 vs profile-2) - filenames +are the same (profile-1), that means that only one set of rules in +memory would be active for particular profile (either setgid or +net_bind_service), but not both. Such problems are hard to debug, so +caution needed while setting configs up. + Setting user passwords ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tools/gate/scripts/020-test-divingbell.sh b/tools/gate/scripts/020-test-divingbell.sh index a65e508..136c18d 100755 --- a/tools/gate/scripts/020-test-divingbell.sh +++ b/tools/gate/scripts/020-test-divingbell.sh @@ -56,7 +56,10 @@ APT_PACKAGE4=less APT_PACKAGE5=python-setuptools APT_PACKAGE6=telnetd EXEC_DIR=/var/${NAME}/exec +# this used in test_overrides to check amount of daemonsets defined +EXPECTED_NUMBER_OF_DAEMONSETS=17 type lshw || apt -y install lshw +type apparmor_parser || apt -y install apparmor nic_info="$(lshw -class network)" physical_nic='' IFS=$'\n' @@ -109,6 +112,7 @@ clean_persistent_files(){ sudo rm -r /var/${NAME} >& /dev/null || true sudo rm -r /etc/sysctl.d/60-${NAME}-* >& /dev/null || true sudo rm -r /etc/security/limits.d/60-${NAME}-* >& /dev/null || true + sudo rm -r /etc/apparmor.d/${NAME}-* >& /dev/null || true _teardown_systemd ${MOUNTS_PATH1} mount _teardown_systemd ${MOUNTS_PATH2} mount _teardown_systemd ${MOUNTS_PATH3} mount @@ -1392,9 +1396,9 @@ test_overrides(){ # Compare against expected number of generated daemonsets daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)" - if [ "${daemonset_count}" != "16" ]; then + if [ "${daemonset_count}" != "${EXPECTED_NUMBER_OF_DAEMONSETS}" ]; then echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}" - echo "Expected 15 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}" + echo "Expected ${EXPECTED_NUMBER_OF_DAEMONSETS} daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}" exit 1 else echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}" @@ -1566,6 +1570,167 @@ test_overrides(){ } +_test_apparmor_profile_added(){ + local profile_file=$1 + local profile_name=$2 + local defaults_path='/var/divingbell/apparmor' + local persist_path='/etc/apparmor.d' + + if [ ! -f "${defaults_path}/${profile_file}" ]; then + return 1 + fi + if [ ! -L "${persist_path}/${profile_file}" ]; then + return 1 + fi + + profile_loaded=$(grep $profile_name /sys/kernel/security/apparmor/profiles || : ) + + if [ -z "$profile_loaded" ]; then + return 1 + fi + return 0 +} + +_test_apparmor_profile_removed(){ + local profile_file=$1 + local profile_name=$2 + local defaults_path='/var/divingbell/apparmor' + local persist_path='/etc/apparmor.d' + + if [ -f "${defaults_path}/${profile_file}" ]; then + return 1 + fi + if [ -L "${persist_path}/${profile_file}" ]; then + return 1 + fi + + profile_loaded=$(grep $profile_name /sys/kernel/security/apparmor/profiles || : ) + + if [ ! -z "$profile_loaded" ]; then + return 1 + fi + + reboot_message_present=$(grep $profile_file /var/run/reboot-required.pkgs || : ) + + if [ -z "$reboot_message_present" ]; then + return 1 + fi + + return 0 +} + +test_apparmor(){ + local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-apparmor.yaml + + #Test1 - check new profile added and loaded + echo "conf: + apparmor: + profiles: + divingbell-profile-1: | + #include + /usr/sbin/profile-1 { + #include + #include + #include + + capability dac_override, + capability dac_read_search, + capability net_bind_service, + capability setgid, + capability setuid, + + /data/www/safe/* r, + deny /data/www/unsafe/* r, + }" > "${overrides_yaml}" + install_base "--values=${overrides_yaml}" + get_container_status apparmor + _test_apparmor_profile_added divingbell-profile-1 profile-1 + echo '[SUCCESS] apparmor test1 passed successfully' >> "${TEST_RESULTS}" + + #Test2 - check new profile added and loaded, profile-1 still exist + echo "conf: + apparmor: + profiles: + divingbell-profile-1: | + #include + /usr/sbin/profile-1 { + #include + #include + #include + + capability dac_override, + capability dac_read_search, + capability net_bind_service, + capability setgid, + capability setuid, + + /data/www/safe/* r, + deny /data/www/unsafe/* r, + } + divingbell-profile-2: | + #include + /usr/sbin/profile-2 { + #include + #include + #include + + capability dac_override, + capability dac_read_search, + capability net_bind_service, + capability setgid, + capability setuid, + + /data/www/safe/* r, + deny /data/www/unsafe/* r, + }" > "${overrides_yaml}" + install_base "--values=${overrides_yaml}" + get_container_status apparmor + _test_apparmor_profile_added divingbell-profile-1 profile-1 + _test_apparmor_profile_added divingbell-profile-2 profile-2 + echo '[SUCCESS] apparmor test2 passed successfully' >> "${TEST_RESULTS}" + + #Test3 - check profile-2 removed, profile-1 still exist + echo "conf: + apparmor: + complain_mode: true + profiles: + divingbell-profile-1: | + #include + /usr/sbin/profile-1 { + #include + #include + #include + + capability dac_override, + capability dac_read_search, + capability net_bind_service, + capability setgid, + capability setuid, + + /data/www/safe/* r, + deny /data/www/unsafe/* r, + }" > "${overrides_yaml}" + install_base "--values=${overrides_yaml}" + get_container_status apparmor + _test_apparmor_profile_added divingbell-profile-1 profile-1 + _test_apparmor_profile_removed divingbell-profile-2 profile-2 + echo '[SUCCESS] apparmor test3 passed successfully' >> "${TEST_RESULTS}" + + #Test4 - check for bad profile input + echo "conf: + apparmor: + profiles: + divingbell-profile-3: | + #include + /usr/sbin/profile-3 { + bad data + }" > "${overrides_yaml}" + install_base "--values=${overrides_yaml}" + get_container_status apparmor expect_failure + _test_clog_msg 'AppArmor parser error for /etc/apparmor.d/divingbell-profile-3 in /etc/apparmor.d/divingbell-profile-3 at line 3: syntax error, unexpected TOK_ID, expecting TOK_MODE' + echo '[SUCCESS] apparmor test4 passed successfully' >> "${TEST_RESULTS}" +} + # initialization init_default_state @@ -1580,6 +1745,7 @@ if [[ -z $SKIP_BASE_TESTS ]]; then test_uamlite test_apt test_exec + test_apparmor fi purge_containers test_overrides