Add new apparmor daemonset

Implemented daemonset that will manage host apparmor profiles.
Tests and documentation added.

demo: https://asciinema.org/a/uQjlWgC4bjI3WkfontmThf8t0

Co-Authored-By: Vladyslav Drok <vdrok@mirantis.com>
Change-Id: I13f7357c15b5c4386a61bba50f097eb434d7f211
This commit is contained in:
Nikita Koshikov 2018-08-29 10:40:30 -07:00 committed by Vladyslav Drok
parent 4ed467e512
commit 606cf35bda
6 changed files with 487 additions and 2 deletions

View File

@ -0,0 +1,137 @@
#!/bin/bash
{{/*
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/}}
set -e
cat <<'EOF' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apparmor_host.sh
{{ include "divingbell.shcommon" . }}
load_flags="-r -W"
{{- if hasKey .Values.conf "apparmor" }}
{{- if hasKey .Values.conf.apparmor "complain_mode" }}
{{- if .Values.conf.apparmor.complain_mode }}
load_flags="$load_flags -C"
{{- end }}
{{- end }}
{{- end }}
load_cmd="apparmor_parser $load_flags"
unload_cmd='apparmor_parser -R'
defaults_path='/var/divingbell/apparmor'
persist_path='/etc/apparmor.d'
declare -A CURRENT_FILENAMES
declare -A SAVED_STATE_FILENAMES
if [ ! -d "${defaults_path}" ]; then
mkdir -p "${defaults_path}"
fi
write_test "${defaults_path}"
write_test "${persist_path}"
save_apparmor_profile(){
local filename="$1"
local data="$2"
CURRENT_FILENAMES["$filename"]=''
#Check if host already had the same filename
if [ ${SAVED_STATE_FILENAMES["$filename"]+_} ]; then
unset SAVED_STATE_FILENAMES["$filename"]
fi
echo -ne "${data}" > ${defaults_path}/${filename}
if [ ! -L ${persist_path}/${filename} ]; then
ln -s ${defaults_path}/${filename} ${persist_path}/${filename}
fi
}
#######################################
#Stage 1
#Collect data
#######################################
#Search for any saved apparmor profiles
pushd $defaults_path
count=$(find . -type f | wc -l)
#Check if directory is non-empty
if [ $count -gt 0 ]; then
for f in $(find . -type f|xargs -n1 basename); do
SAVED_STATE_FILENAMES[$f]=''
done
fi
#######################################
#Stage 2
#Save new apparmor profiles
#######################################
{{- if hasKey .Values.conf "apparmor" }}
{{- if hasKey .Values.conf.apparmor "profiles" }}
{{- range $filename, $value := .Values.conf.apparmor.profiles }}
save_apparmor_profile {{ $filename | squote }} {{ $value | squote }}
{{- end }}
{{- end }}
{{- end }}
#######################################
#Stage 3
#Clean stale apparmor profiles
#######################################
#If hash is not empty - there are old filenames that need to be handled
if [ ${#SAVED_STATE_FILENAMES[@]} -gt 0 ]; then
for filename in ${!SAVED_STATE_FILENAMES[@]}; do
#Unload any previously applied apparmor profiles which are now absent
$unload_cmd ${defaults_path}/${filename} || die "Problem unloading profile ${defaults_path}/${filename}"
if [ -L ${persist_path}/${filename} ]; then
unlink ${persist_path}/${filename}
fi
rm -f ${defaults_path}/${filename}
# log/append the stale profiles that require eventual reboot
echo "apparmor: stale profile ${defaults_path}/${filename}" >> /var/run/reboot-required.pkgs
unset SAVED_STATE_FILENAMES["$filename"]
done
# mark node as needing eventual reboot
echo '*** System restart required ***' > /var/run/reboot-required
fi
#######################################
#Stage 4
#Install/update new apparmor profiles
#Save new apparmor profiles
#######################################
for filename in ${!CURRENT_FILENAMES[@]}; do
$load_cmd ${persist_path}/${filename} || die "Problem loading ${persist_path}/${filename}"
done
exit 0
EOF
chmod 755 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apparmor_host.sh
chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/apparmor_host.sh
sleep 1
echo 'INFO Putting the daemon to sleep.'
while [ 1 ]; do
sleep 300
done
exit 0

View File

@ -0,0 +1,69 @@
{{/*
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/}}
{{- define "divingbell.daemonset.apparmor" }}
{{- $daemonset := index . 0 }}
{{- $secretName := index . 1 }}
{{- $envAll := index . 2 }}
{{- with $envAll }}
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: {{ $daemonset }}
spec:
{{ tuple $envAll $daemonset | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
template:
metadata:
labels:
{{ list $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
hostNetwork: true
hostPID: true
hostIPC: true
containers:
- name: {{ $daemonset }}
image: {{ .Values.images.divingbell }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.apparmor | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
command:
- /tmp/{{ $daemonset }}.sh
volumeMounts:
- name: rootfs-{{ $daemonset }}
mountPath: {{ .Values.conf.chroot_mnt_path }}
- name: {{ $secretName }}
mountPath: /tmp/{{ $daemonset }}.sh
subPath: {{ $daemonset }}
readOnly: true
securityContext:
privileged: true
volumes:
- name: rootfs-{{ $daemonset }}
hostPath:
path: /
- name: {{ $secretName }}
secret:
secretName: {{ $secretName }}
defaultMode: 0555
{{- end }}
{{- end }}
{{- if .Values.manifests.daemonset_apparmor }}
{{- $daemonset := "apparmor" }}
{{- $secretName := "divingbell-apparmor" }}
{{- $daemonset_yaml := list $daemonset $secretName . | include "divingbell.daemonset.apparmor" | toString | fromYaml }}
{{- $secret_include := "divingbell.secret.apparmor" }}
{{- list $daemonset $daemonset_yaml $secret_include $secretName . | include "helm-toolkit.utils.daemonset_overrides" }}
{{- end }}

View File

@ -0,0 +1,29 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "divingbell.secret.apparmor" }}
{{- $secretName := index . 0 }}
{{- $envAll := index . 1 }}
{{- with $envAll }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
data:
apparmor: {{ tuple "bin/_apparmor.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
{{- end }}
{{- end }}

View File

@ -127,6 +127,13 @@ pod:
max_unavailable: 100%
resources:
enabled: false
apparmor:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
ethtool:
limits:
memory: "128Mi"
@ -193,3 +200,4 @@ manifests:
daemonset_apt: true
daemonset_perm: true
daemonset_exec: true
daemonset_apparmor: true

View File

@ -250,9 +250,85 @@ The following set of options are not yet implemeneted::
failing script should be retried. Failed exec count does not persist
through pod/node restart. Default value is ``infinite``.
apparmor
^^^^^^^^
Used to manage host level apparmor profiles/rules, Ex::
conf:
apparmor:
complain_mode: "true"
profiles:
profile-1: |
#include <tunables/global>
/usr/sbin/profile-1 {
#include <abstractions/apache2-common>
#include <abstractions/base>
#include <abstractions/nis>
capability dac_override,
capability dac_read_search,
capability net_bind_service,
capability setgid,
capability setuid,
/data/www/safe/* r,
deny /data/www/unsafe/* r,
}
profile-2: |
#include <tunables/global>
/usr/sbin/profile-2 {
#include <abstractions/apache2-common>
#include <abstractions/base>
#include <abstractions/nis>
capability dac_override,
capability dac_read_search,
capability net_bind_service,
capability setgid,
capability setuid,
/data/www/safe/* r,
deny /data/www/unsafe/* r,
}
Operations
----------
Setting apparmor profiles
^^^^^^^^^^^^^^^^^^^^^^^^^
The way apparmor loading/unloading implemented is through saving
settings to a file and than running ``apparmor_parser`` command.
The daemonset supports both enforcement and complain mode,
enforcement being the default. To request complain mode for the
profiles, add ``complain_mode: "true"`` nested under apparmor entry.
It's easy to mess up host with rules, if profile names would
distinguish from file content. Ex::
conf:
apparmor:
profiles:
profile-1: |
#include <tunables/global>
/usr/sbin/profile-1 {
#include <abstractions/base>
capability setgid,
}
profile-2: |
#include <tunables/global>
/usr/sbin/profile-1 {
#include <abstractions/base>
capability net_bind_service,
}
Even when profiles are different (profile-1 vs profile-2) - filenames
are the same (profile-1), that means that only one set of rules in
memory would be active for particular profile (either setgid or
net_bind_service), but not both. Such problems are hard to debug, so
caution needed while setting configs up.
Setting user passwords
^^^^^^^^^^^^^^^^^^^^^^

View File

@ -56,7 +56,10 @@ APT_PACKAGE4=less
APT_PACKAGE5=python-setuptools
APT_PACKAGE6=telnetd
EXEC_DIR=/var/${NAME}/exec
# this used in test_overrides to check amount of daemonsets defined
EXPECTED_NUMBER_OF_DAEMONSETS=17
type lshw || apt -y install lshw
type apparmor_parser || apt -y install apparmor
nic_info="$(lshw -class network)"
physical_nic=''
IFS=$'\n'
@ -109,6 +112,7 @@ clean_persistent_files(){
sudo rm -r /var/${NAME} >& /dev/null || true
sudo rm -r /etc/sysctl.d/60-${NAME}-* >& /dev/null || true
sudo rm -r /etc/security/limits.d/60-${NAME}-* >& /dev/null || true
sudo rm -r /etc/apparmor.d/${NAME}-* >& /dev/null || true
_teardown_systemd ${MOUNTS_PATH1} mount
_teardown_systemd ${MOUNTS_PATH2} mount
_teardown_systemd ${MOUNTS_PATH3} mount
@ -1392,9 +1396,9 @@ test_overrides(){
# Compare against expected number of generated daemonsets
daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)"
if [ "${daemonset_count}" != "16" ]; then
if [ "${daemonset_count}" != "${EXPECTED_NUMBER_OF_DAEMONSETS}" ]; then
echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}"
echo "Expected 15 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
echo "Expected ${EXPECTED_NUMBER_OF_DAEMONSETS} daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
exit 1
else
echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}"
@ -1566,6 +1570,167 @@ test_overrides(){
}
_test_apparmor_profile_added(){
local profile_file=$1
local profile_name=$2
local defaults_path='/var/divingbell/apparmor'
local persist_path='/etc/apparmor.d'
if [ ! -f "${defaults_path}/${profile_file}" ]; then
return 1
fi
if [ ! -L "${persist_path}/${profile_file}" ]; then
return 1
fi
profile_loaded=$(grep $profile_name /sys/kernel/security/apparmor/profiles || : )
if [ -z "$profile_loaded" ]; then
return 1
fi
return 0
}
_test_apparmor_profile_removed(){
local profile_file=$1
local profile_name=$2
local defaults_path='/var/divingbell/apparmor'
local persist_path='/etc/apparmor.d'
if [ -f "${defaults_path}/${profile_file}" ]; then
return 1
fi
if [ -L "${persist_path}/${profile_file}" ]; then
return 1
fi
profile_loaded=$(grep $profile_name /sys/kernel/security/apparmor/profiles || : )
if [ ! -z "$profile_loaded" ]; then
return 1
fi
reboot_message_present=$(grep $profile_file /var/run/reboot-required.pkgs || : )
if [ -z "$reboot_message_present" ]; then
return 1
fi
return 0
}
test_apparmor(){
local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-apparmor.yaml
#Test1 - check new profile added and loaded
echo "conf:
apparmor:
profiles:
divingbell-profile-1: |
#include <tunables/global>
/usr/sbin/profile-1 {
#include <abstractions/apache2-common>
#include <abstractions/base>
#include <abstractions/nis>
capability dac_override,
capability dac_read_search,
capability net_bind_service,
capability setgid,
capability setuid,
/data/www/safe/* r,
deny /data/www/unsafe/* r,
}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apparmor
_test_apparmor_profile_added divingbell-profile-1 profile-1
echo '[SUCCESS] apparmor test1 passed successfully' >> "${TEST_RESULTS}"
#Test2 - check new profile added and loaded, profile-1 still exist
echo "conf:
apparmor:
profiles:
divingbell-profile-1: |
#include <tunables/global>
/usr/sbin/profile-1 {
#include <abstractions/apache2-common>
#include <abstractions/base>
#include <abstractions/nis>
capability dac_override,
capability dac_read_search,
capability net_bind_service,
capability setgid,
capability setuid,
/data/www/safe/* r,
deny /data/www/unsafe/* r,
}
divingbell-profile-2: |
#include <tunables/global>
/usr/sbin/profile-2 {
#include <abstractions/apache2-common>
#include <abstractions/base>
#include <abstractions/nis>
capability dac_override,
capability dac_read_search,
capability net_bind_service,
capability setgid,
capability setuid,
/data/www/safe/* r,
deny /data/www/unsafe/* r,
}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apparmor
_test_apparmor_profile_added divingbell-profile-1 profile-1
_test_apparmor_profile_added divingbell-profile-2 profile-2
echo '[SUCCESS] apparmor test2 passed successfully' >> "${TEST_RESULTS}"
#Test3 - check profile-2 removed, profile-1 still exist
echo "conf:
apparmor:
complain_mode: true
profiles:
divingbell-profile-1: |
#include <tunables/global>
/usr/sbin/profile-1 {
#include <abstractions/apache2-common>
#include <abstractions/base>
#include <abstractions/nis>
capability dac_override,
capability dac_read_search,
capability net_bind_service,
capability setgid,
capability setuid,
/data/www/safe/* r,
deny /data/www/unsafe/* r,
}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apparmor
_test_apparmor_profile_added divingbell-profile-1 profile-1
_test_apparmor_profile_removed divingbell-profile-2 profile-2
echo '[SUCCESS] apparmor test3 passed successfully' >> "${TEST_RESULTS}"
#Test4 - check for bad profile input
echo "conf:
apparmor:
profiles:
divingbell-profile-3: |
#include <tunables/global>
/usr/sbin/profile-3 {
bad data
}" > "${overrides_yaml}"
install_base "--values=${overrides_yaml}"
get_container_status apparmor expect_failure
_test_clog_msg 'AppArmor parser error for /etc/apparmor.d/divingbell-profile-3 in /etc/apparmor.d/divingbell-profile-3 at line 3: syntax error, unexpected TOK_ID, expecting TOK_MODE'
echo '[SUCCESS] apparmor test4 passed successfully' >> "${TEST_RESULTS}"
}
# initialization
init_default_state
@ -1580,6 +1745,7 @@ if [[ -z $SKIP_BASE_TESTS ]]; then
test_uamlite
test_apt
test_exec
test_apparmor
fi
purge_containers
test_overrides