From ae819b9a3bb9f9319f0d570cd06705a869425550 Mon Sep 17 00:00:00 2001 From: Craig Anderson Date: Wed, 22 Nov 2017 22:29:13 +0000 Subject: [PATCH] Add label prioritization Altered daemonset scheduling to determonistically ensure one and only one daemonset is schedule to each node. This is done via implicit label override prioritization. If nodes contain multiple labels, a given daemonset will always schedule to whichever label was the last to be defined in overrides yaml. Change-Id: Ib90f36f27e3bcd50d017262c07317aa3a64464bb --- README.md | 9 ++- divingbell/templates/daemonsets.yaml | 8 ++- divingbell/tools/gate/test.sh | 93 +++++++++++++++++++++++----- 3 files changed, 86 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 6acaca7..eee4c40 100644 --- a/README.md +++ b/README.md @@ -164,11 +164,10 @@ precedence and are used for that node. The label overrides are not used in this case. This is especially important to note if you are defining new host overrides for a node that is already consuming matching label overrides, as defining a host override would make those label overrides no longer apply. -2. Daemonsets are generated regardless of the current state of the environment. -Ex: If your environment consists of a single node that matches a host override, -the chart will still generate a default daemonset which would fail to schedule -in this example. Likewise if the host or label in the override return no -candidates, these would also fail to schedule. +2. In the event of label conflicts, the last applicable label override defined +takes precedence. In this example, overrides defined for "another_label" would +take precedence and be applied to nodes that contained both of the defined +labels. Recorded Demo ------------- diff --git a/divingbell/templates/daemonsets.yaml b/divingbell/templates/daemonsets.yaml index 89ff97a..dca235d 100644 --- a/divingbell/templates/daemonsets.yaml +++ b/divingbell/templates/daemonsets.yaml @@ -63,6 +63,7 @@ {{- end }} {{- if eq $type "labels" }} + {{- set $.Values "__label_list" . }} {{- range $label_data := . }} # dictionary that will contain all info needed to generate this # iteration of the daemonset. @@ -86,8 +87,10 @@ {{- $list_aggregate := list $label_dict }} {{- set $.Values.__current_label "matchExpressions" $list_aggregate }} - # Do not schedule to any other specified labels - {{- $other_labels := without $type_data $label_data }} + # Do not schedule to other specified labels, with higher + # precedence as the list position increases. Last defined label + # is highest priority. + {{- $other_labels := without $.Values.__label_list $label_data }} {{- range $label_data2 := $other_labels }} {{- $label_dict := omit $label_data2.label "NULL" }} @@ -96,6 +99,7 @@ {{- $list_aggregate := append $.Values.__current_label.matchExpressions $label_dict }} {{- set $.Values.__current_label "matchExpressions" $list_aggregate }} {{- end }} + {{- set $.Values "__label_list" $other_labels }} # Do not schedule to any other specified hosts {{- range $type, $type_data := $val }} diff --git a/divingbell/tools/gate/test.sh b/divingbell/tools/gate/test.sh index 26093cf..6c12ef0 100755 --- a/divingbell/tools/gate/test.sh +++ b/divingbell/tools/gate/test.sh @@ -159,7 +159,8 @@ get_container_status(){ echo "${CLOGS}" exit 1 fi - elif [ "${status}" = 'INFO Putting the daemon to sleep.' ]; then + elif [ "${status}" = 'INFO Putting the daemon to sleep.' ] || + [ "${status}" = 'DEBUG + exit 0' ]; then if [ "${2}" = 'expect_failure' ]; then echo 'Expected pod to die with error, but pod completed successfully' echo 'pod logs:' @@ -580,24 +581,15 @@ test_overrides(){ # TODO: Implement more robust tests that do not depend on match expression # ordering. - # Verify generated affinity for one of the daemonset labels + # Verify generated affinity for test_label echo "${tc_output}" | grep ' spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: another_label - operator: In - values: - - "another_value" - - key: compute_type - operator: NotIn - values: - - "dpdk" - - "sriov" - key: test_label - operator: NotIn + operator: In values: - "test_value" - key: kubernetes.io/hostname @@ -616,6 +608,73 @@ test_overrides(){ echo '[SUCCESS] overrides test 2 passed successfully' >> "${TEST_RESULTS}" || (echo '[FAILURE] overrides test 2 failed' && exit 1) + # Verify generated affinity for another_label + echo "${tc_output}" | grep ' spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: another_label + operator: In + values: + - "another_value" + - key: test_label + operator: NotIn + values: + - "test_value" + - key: kubernetes.io/hostname + operator: NotIn + values: + - "superhost" + - key: kubernetes.io/hostname + operator: NotIn + values: + - "helm1" + - key: kubernetes.io/hostname + operator: NotIn + values: + - "specialhost" + hostNetwork: true' && + echo '[SUCCESS] overrides test 3 passed successfully' >> "${TEST_RESULTS}" || + (echo '[FAILURE] overrides test 3 failed' && exit 1) + + # Verify generated affinity for compute_type + echo "${tc_output}" | grep ' spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: compute_type + operator: In + values: + - "dpdk" + - "sriov" + - key: another_label + operator: NotIn + values: + - "another_value" + - key: test_label + operator: NotIn + values: + - "test_value" + - key: kubernetes.io/hostname + operator: NotIn + values: + - "superhost" + - key: kubernetes.io/hostname + operator: NotIn + values: + - "helm1" + - key: kubernetes.io/hostname + operator: NotIn + values: + - "specialhost" + hostNetwork: true' && + echo '[SUCCESS] overrides test 4 passed successfully' >> "${TEST_RESULTS}" || + (echo '[FAILURE] overrides test 4 failed' && exit 1) + # Verify generated affinity for one of the daemonset hosts echo "${tc_output}" | grep ' spec: affinity: @@ -629,8 +688,8 @@ test_overrides(){ - "soup" - "chips" hostNetwork: true' && - echo '[SUCCESS] overrides test 3 passed successfully' >> "${TEST_RESULTS}" || - (echo '[FAILURE] overrides test 3 failed' && exit 1) + echo '[SUCCESS] overrides test 5 passed successfully' >> "${TEST_RESULTS}" || + (echo '[FAILURE] overrides test 5 failed' && exit 1) # Verify generated affinity for one of the daemonset defaults echo "${tc_output}" | grep ' spec: @@ -665,8 +724,8 @@ test_overrides(){ values: - "test_value" hostNetwork: true' && - echo '[SUCCESS] overrides test 4 passed successfully' >> "${TEST_RESULTS}" || - (echo '[FAILURE] overrides test 4 failed' && exit 1) + echo '[SUCCESS] overrides test 6 passed successfully' >> "${TEST_RESULTS}" || + (echo '[FAILURE] overrides test 6 failed' && exit 1) overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-functional.yaml key1_override_val=0 @@ -686,7 +745,7 @@ test_overrides(){ get_container_status sysctl _test_sysctl_default $SYSCTL_KEY1 $key1_override_val _test_sysctl_default $SYSCTL_KEY2 $key2_non_override_val - echo '[SUCCESS] overrides test 5 passed successfully' >> "${TEST_RESULTS}" + echo '[SUCCESS] overrides test 7 passed successfully' >> "${TEST_RESULTS}" }