From 918da6d055be6005246d4042f6accab8e259f9ee Mon Sep 17 00:00:00 2001 From: Phil Sphicas Date: Thu, 5 Nov 2020 21:03:55 +0000 Subject: [PATCH] Avoid rbd unmap failure; use HostToContainer mountPropagation The divingbell pods use a hostPath volume for the root filesystem. Because this mount includes /var/lib/kubelet, the pod holds a reference to every volume mounted by every pod on the same host. The most visible case where this causes a problem is the termination of a pod that uses a ceph-backed PVCs. When kubelet tries to unmap the rbd device, it is unable to do so, manifesting in the kubelet logs as: rbd: unmap failed: (16) Device or resource busy This change sets the mountPropagation to HostToContainer for the rootfs volume, so that the divingbell pods will not prevent kubelet from releasing these devices. https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation Change-Id: I6e91fb9b9d7cbe852c5e6dc8b7224d6085175590 --- divingbell/templates/daemonset-apparmor.yaml | 1 + divingbell/templates/daemonset-apt.yaml | 1 + divingbell/templates/daemonset-ethtool.yaml | 1 + divingbell/templates/daemonset-exec.yaml | 1 + divingbell/templates/daemonset-limits.yaml | 1 + divingbell/templates/daemonset-mounts.yaml | 1 + divingbell/templates/daemonset-perm.yaml | 1 + divingbell/templates/daemonset-sysctl.yaml | 1 + divingbell/templates/daemonset-uamlite.yaml | 1 + 9 files changed, 9 insertions(+) diff --git a/divingbell/templates/daemonset-apparmor.yaml b/divingbell/templates/daemonset-apparmor.yaml index d8510b4..56bd4d0 100644 --- a/divingbell/templates/daemonset-apparmor.yaml +++ b/divingbell/templates/daemonset-apparmor.yaml @@ -58,6 +58,7 @@ spec: mountPath: /tmp - name: rootfs-{{ $daemonset }} mountPath: {{ .Values.conf.chroot_mnt_path }} + mountPropagation: HostToContainer - name: {{ $secretName }} mountPath: /tmp/{{ $daemonset }}.sh subPath: {{ $daemonset }} diff --git a/divingbell/templates/daemonset-apt.yaml b/divingbell/templates/daemonset-apt.yaml index a628c24..c515740 100644 --- a/divingbell/templates/daemonset-apt.yaml +++ b/divingbell/templates/daemonset-apt.yaml @@ -58,6 +58,7 @@ spec: mountPath: /tmp - name: rootfs-{{ $daemonset }} mountPath: {{ .Values.conf.chroot_mnt_path }} + mountPropagation: HostToContainer - name: {{ $secretName }} mountPath: /tmp/{{ $daemonset }}.sh subPath: {{ $daemonset }} diff --git a/divingbell/templates/daemonset-ethtool.yaml b/divingbell/templates/daemonset-ethtool.yaml index c79e35f..3ff53d1 100644 --- a/divingbell/templates/daemonset-ethtool.yaml +++ b/divingbell/templates/daemonset-ethtool.yaml @@ -58,6 +58,7 @@ spec: mountPath: /tmp - name: rootfs-{{ $daemonset }} mountPath: {{ .Values.conf.chroot_mnt_path }} + mountPropagation: HostToContainer - name: {{ $secretName }} mountPath: /tmp/{{ $daemonset }}.sh subPath: {{ $daemonset }} diff --git a/divingbell/templates/daemonset-exec.yaml b/divingbell/templates/daemonset-exec.yaml index 95a989a..243a7dd 100644 --- a/divingbell/templates/daemonset-exec.yaml +++ b/divingbell/templates/daemonset-exec.yaml @@ -58,6 +58,7 @@ spec: mountPath: /tmp - name: rootfs-{{ $daemonset }} mountPath: {{ .Values.conf.chroot_mnt_path }} + mountPropagation: HostToContainer - name: {{ $secretName }} mountPath: /tmp/{{ $daemonset }}.sh subPath: {{ $daemonset }} diff --git a/divingbell/templates/daemonset-limits.yaml b/divingbell/templates/daemonset-limits.yaml index 15d084c..b695a32 100644 --- a/divingbell/templates/daemonset-limits.yaml +++ b/divingbell/templates/daemonset-limits.yaml @@ -58,6 +58,7 @@ spec: mountPath: /tmp - name: rootfs-{{ $daemonset }} mountPath: {{ .Values.conf.chroot_mnt_path }} + mountPropagation: HostToContainer - name: {{ $secretName }} mountPath: /tmp/{{ $daemonset }}.sh subPath: {{ $daemonset }} diff --git a/divingbell/templates/daemonset-mounts.yaml b/divingbell/templates/daemonset-mounts.yaml index e3604a1..f6bd82a 100644 --- a/divingbell/templates/daemonset-mounts.yaml +++ b/divingbell/templates/daemonset-mounts.yaml @@ -58,6 +58,7 @@ spec: mountPath: /tmp - name: rootfs-{{ $daemonset }} mountPath: {{ .Values.conf.chroot_mnt_path }} + mountPropagation: HostToContainer - name: {{ $secretName }} mountPath: /tmp/{{ $daemonset }}.sh subPath: {{ $daemonset }} diff --git a/divingbell/templates/daemonset-perm.yaml b/divingbell/templates/daemonset-perm.yaml index 0e02cd0..a3e29d2 100644 --- a/divingbell/templates/daemonset-perm.yaml +++ b/divingbell/templates/daemonset-perm.yaml @@ -58,6 +58,7 @@ spec: mountPath: /tmp - name: rootfs-{{ $daemonset }} mountPath: {{ .Values.conf.chroot_mnt_path }} + mountPropagation: HostToContainer - name: {{ $secretName }} mountPath: /tmp/{{ $daemonset }}.sh subPath: {{ $daemonset }} diff --git a/divingbell/templates/daemonset-sysctl.yaml b/divingbell/templates/daemonset-sysctl.yaml index 278b9c0..b3f45ab 100644 --- a/divingbell/templates/daemonset-sysctl.yaml +++ b/divingbell/templates/daemonset-sysctl.yaml @@ -58,6 +58,7 @@ spec: mountPath: /tmp - name: rootfs-{{ $daemonset }} mountPath: {{ .Values.conf.chroot_mnt_path }} + mountPropagation: HostToContainer - name: {{ $secretName }} mountPath: /tmp/{{ $daemonset }}.sh subPath: {{ $daemonset }} diff --git a/divingbell/templates/daemonset-uamlite.yaml b/divingbell/templates/daemonset-uamlite.yaml index 2dbdd1f..13df7f8 100644 --- a/divingbell/templates/daemonset-uamlite.yaml +++ b/divingbell/templates/daemonset-uamlite.yaml @@ -54,6 +54,7 @@ spec: volumeMounts: - name: rootfs-{{ $daemonset }} mountPath: {{ .Values.conf.chroot_mnt_path }} + mountPropagation: HostToContainer - name: {{ $secretName }} mountPath: /tmp/{{ $daemonset }}.sh subPath: {{ $daemonset }}