Add rerun support for perm module

- Adds the ability to rerun divingbell-perm at specified interval.

- Adds the ability to specify a rerun policy of
  'always', 'never', 'once_successfully'. Default value is 'always'.

Demo: https://asciinema.org/a/220289

Change-Id: I3909b4d92f8e2bdb0d826ca1cfbd62f937c2532d

3 files changed, 158 insertions, 34 deletions

diff --git a/divingbell/templates/bin/_perm.sh.tpl b/divingbell/templates/bin/_perm.sh.tpl
index 6aac03e..58b87af 100644
--- a/divingbell/templates/bin/_perm.sh.tpl
+++ b/divingbell/templates/bin/_perm.sh.tpl
@@ -16,6 +16,8 @@
 # limitations under the License.
 */}}
 
+{{- $perm_loop_sleep_interval := 60 }}
+
 set -e
 
 cat <<'EOF' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/perm_host.sh
@@ -82,54 +84,127 @@ add_single_perm(){
 
 }
 
-{{- range $perm := .Values.conf.perm }}
-add_perm {{ $perm.path | squote }} {{ $perm.owner | squote }} {{ $perm.group | squote }} {{ $perm.permissions | squote }}
+revert_perm(){
+# Revert
+  prev_files="$(find "${backup_path}" -type f ! -name last_run_timestamp)"
+  if [ -n "${prev_files}" ]; then
+    basename -a ${prev_files} | sort > /tmp/prev_perm
+    echo "${applied_perm}" | sort > /tmp/curr_perm
+    log.DEBUG /tmp/prev_perm
+    log.DEBUG /tmp/curr_perm
+    revert_list="$(comm -23 /tmp/prev_perm /tmp/curr_perm)"
+    IFS=$'\n'
+    for o_perm in ${revert_list}; do
+      first=1
+      while IFS=' ' read -r a1 a2; do
+        if [ "$first" -eq 1 ]; then
+          $(chmod $a1 $a2)
+          first=0
+        else
+          $(chown $a1 $a2)
+        fi
+      done < "${backup_path}/${o_perm}"
+
+      rm "${backup_path}/${o_perm}"
+      log.INFO "Reverted permissions and owner: ${backup_path}/${o_perm}"
+    done
+  fi
+}
+
+{{- $_ := set $.Values "__rerun_policy" "always" }}
+{{- if hasKey .Values.conf "perm" }}
+{{- if hasKey .Values.conf.perm "rerun_policy" }}
+  {{- if and (not (eq .Values.conf.perm.rerun_policy "always")) (not (eq .Values.conf.perm.rerun_policy "never")) (not (eq .Values.conf.perm.rerun_policy "once_successfully")) }}
+    {{- fail (print "BAD 'rerun_policy' Got '" .Values.conf.perm.rerun_policy "', but expected 'always', 'never', or 'once_successfully'.") }}
+  {{- end }}
+  {{- $_ := set $.Values "__rerun_policy" .Values.conf.perm.rerun_policy }}
 {{- end }}
 
-log.INFO "Applied: ${applied_perm}"
+{{- $_ := set $.Values "__rerun_interval" "infinite" }}
+{{- if hasKey .Values.conf.perm "rerun_interval" }}
+{{- $_ := set $.Values "__rerun_interval" .Values.conf.perm.rerun_interval }}
+
+  {{- if not (eq (.Values.conf.perm.rerun_interval | toString) "infinity") }}
+    {{- if lt (.Values.conf.perm.rerun_interval | int) $perm_loop_sleep_interval }}
+      {{- fail (print "BAD 'rerun_interval' Got '" $.Values.__rerun_interval "', but expected >= '" $perm_loop_sleep_interval "'.") }}
+    {{- end }}
+    {{- if not (eq $.Values.__rerun_policy "always") }}
+      {{- fail (print "BAD COMBINATION: Must use 'rerun_policy' of 'always' when defining a finite 'retry_interval'. Got 'rerun_policy' of '" $.Values.__rerun_policy "' and 'retry_interval' of '" $.Values.__rerun_interval "'.") }}
+    {{- end }}
+  {{- end }}
+  {{- $_ := set $.Values "__rerun_interval" .Values.conf.perm.rerun_interval }}
+{{- end }}
 
-# Revert
-prev_files="$(find "${backup_path}" -type f)"
-if [ -n "${prev_files}" ]; then
-  basename -a ${prev_files} | sort > /tmp/prev_perm
-  echo "${applied_perm}" | sort > /tmp/curr_perm
-  log.DEBUG /tmp/prev_perm
-  log.DEBUG /tmp/curr_perm
-  revert_list="$(comm -23 /tmp/prev_perm /tmp/curr_perm)"
-  IFS=$'\n'
-  for o_perm in ${revert_list}; do
-    first=1
-    while IFS=' ' read -r a1 a2; do
-      if [ "$first" -eq 1 ]; then
-        $(chmod $a1 $a2)
-        first=0
-      else
-        $(chown $a1 $a2)
-      fi
-    done < "${backup_path}/${o_perm}"
-
-    rm "${backup_path}/${o_perm}"
-    log.INFO "Reverted permissions and owner: ${backup_path}/${o_perm}"
-  done
+{{- if hasKey .Values.conf.perm "rerun_policy" }}
+      {{- if and (not (eq $.Values.__rerun_policy "always")) (not (eq $.Values.__rerun_policy "never")) (not (eq $.Values.__rerun_policy "once_successfully")) }}
+        {{- fail (print "BAD 'rerun_policy' : Got '" $.Values.__rerun_policy "', but expected 'always', 'never', or 'once_successfully'.") }}
+      {{- end }}
+{{- end }}
+
+cd "${backup_path}"
+
+{{- $_ := set $.Values "__values_hash" list }}
+{{- $hash := $.Values.__values_hash | toString | sha256sum }}
+
+hash={{ $hash | squote }}
+if [ ! -d "${hash}" ]; then
+  mkdir -p "${hash}"
+fi
+
+# check rerun policy
+hash_check=fail
+if  [[ {{ $.Values.__rerun_policy }} = always ]] || \
+    [[ ! -f ${hash}/exit_code ]] || \
+   ([[ {{ $.Values.__rerun_policy }} = once_successfully ]] && \
+    [[ $(cat ${hash}/exit_code) != 0 ]]); then
+  hash_check=pass
+fi
+# check rerun interval
+interval_check=fail
+if  [[ ! -f ${hash}/last_run_timestamp ]] || [[ ! -f ${hash}/exit_code ]]; then
+  interval_check=pass
+elif [[ $(cat ${hash}/exit_code) = 0 ]]; then
+  if [[ {{ $.Values.__rerun_interval }} = infinite ]]; then
+    interval_check=pass
+  elif [[ $(date +"%s") -ge $(($(cat ${hash}/last_run_timestamp) + {{ $.Values.__rerun_interval }})) ]]; then
+    interval_check=pass
+  fi
 fi
+if [[ $hash_check = pass ]] && [[ $interval_check = pass ]]; then
+  if [[ -f ${hash}/exit_code ]]; then
+    # remove previous run record, in case this run is interrupted
+    rm ${hash}/exit_code
+  fi
+ # write timestamp at beginning of execution
+ log.INFO 'All permissions successfully applied on this node.'
+ echo $(date +"%s") > "${hash}/last_run_timestamp"
+
+ {{- range $perm := .Values.conf.perm.paths }}
+ add_perm {{ $perm.path | squote }} {{ $perm.owner | squote }} {{ $perm.group | squote }} {{ $perm.permissions | squote }}
+ {{- end }}
+ log.INFO "Applied: ${applied_perm}"
 
-if [ -n "${curr_settings}" ]; then
+ revert_perm
+
+ if [ -n "${curr_settings}" ]; then
   log.INFO 'All permissions successfully applied on this node.'
-else
+ else
   log.WARN 'No permissions overrides defined for this node.'
+ fi
 fi
 
+echo 0 > "${hash}/exit_code"
 exit 0
+{{- end}}
 EOF
 
 chmod 755 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/perm_host.sh
-chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/perm_host.sh
-
-sleep 1
-echo 'INFO Putting the daemon to sleep.'
 
-while [ 1 ]; do
-  sleep 300
+while true; do
+  chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/perm_host.sh
+  sleep 2
+  echo 'INFO Putting the daemon to sleep.'
+  sleep {{ $perm_loop_sleep_interval }}
 done
 
 exit 0
diff --git a/divingbell/values.yaml b/divingbell/values.yaml
index 84fb1e9..931fc6b 100644
--- a/divingbell/values.yaml
+++ b/divingbell/values.yaml
@@ -32,6 +32,10 @@ conf:
     - nis
     - ntpdate
 #  perm:
+#    rerun_policy: always
+#    86400 = 1 day
+#    rerun_interval: 86400
+#    paths:
 #    -
 #      path: '/boot/System.map-*'
 #      owner: 'root'
diff --git a/tools/gate/scripts/020-test-divingbell.sh b/tools/gate/scripts/020-test-divingbell.sh
index 136c18d..c8e593d 100755
--- a/tools/gate/scripts/020-test-divingbell.sh
+++ b/tools/gate/scripts/020-test-divingbell.sh
@@ -435,6 +435,7 @@ test_perm(){
   local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}.yaml
   echo "conf:
   perm:
+    paths:
     -
       path: ${p_test_file1}
       owner: 'root'
@@ -452,6 +453,7 @@ test_perm(){
   echo "[SUCCESS] Positive test for perm passed successfully" >> "${TEST_RESULTS}"
   echo "conf:
   perm:
+    paths:
     -
       path: ${p_test_file1}
       owner: 'root'
@@ -462,6 +464,49 @@ test_perm(){
   _test_perm_value ${p_test_file1} root shadow 640
   _test_perm_value ${p_test_file2} ${p_test_file2##*.} ${p_test_file2##*.} 777
   echo "[SUCCESS] Backup test for perm passed successfully" >> "${TEST_RESULTS}"
+  # Test invalid rerun_interval (too short)
+  echo "conf:
+  perm:
+    rerun_interval: 30
+    paths:
+    -
+      path: ${p_test_file1}
+      owner: 'root'
+      group: 'shadow'
+      permissions: '0640'" > "${overrides_yaml}"
+  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .rerun_interval. Got' || \
+    (echo "[FAIL] perm test invalid rerun_interval value did not receive expected 'BAD .rerun_interval. Got' error" && exit 1)
+  echo '[SUCCESS] perm test invalid rerun_interval passed successfully' >> "${TEST_RESULTS}"
+  # Test invalid rerun_interval combination
+  echo "conf:
+  perm:
+    rerun_interval: 60
+    rerun_policy: once_successfully
+    paths:
+    -
+      path: ${p_test_file1}
+      owner: 'root'
+      group: 'shadow'
+      permissions: '0640'" > "${overrides_yaml}"
+  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD COMBINATION' || \
+    (echo "[FAIL] perm invalid rerun_interval combination did not receive expected 'BAD COMBINATION' error" && exit 1)
+  echo '[SUCCESS] perm invalid rerun_interval combination passed successfully' >> "${TEST_RESULTS}"
+  # test rerun_interval
+  echo "conf:
+  perm:
+    rerun_interval: 60
+    paths:
+    -
+      path: ${p_test_file1}
+      owner: 'root'
+      group: 'shadow'
+      permissions: '0640'" > "${overrides_yaml}"
+  install_base "--values=${overrides_yaml}"
+  get_container_status perm
+  sleep 72
+  get_container_status perm
+  _test_perm_value ${p_test_file1} root shadow 640
+  echo '[SUCCESS] perm rerun_interval passed successfully' >> "${TEST_RESULTS}"
   _perm_teardown
 }