Add label prioritization

Altered daemonset scheduling to determonistically ensure one and
only one daemonset is schedule to each node. This is done via
implicit label override prioritization. If nodes contain multiple
labels, a given daemonset will always schedule to whichever label
was the last to be defined in overrides yaml.

Change-Id: Ib90f36f27e3bcd50d017262c07317aa3a64464bb
This commit is contained in:
Craig Anderson 2017-11-22 22:29:13 +00:00
parent 6d2f967ce4
commit ae819b9a3b
3 changed files with 86 additions and 24 deletions

View File

@ -164,11 +164,10 @@ precedence and are used for that node. The label overrides are not used in this
case. This is especially important to note if you are defining new host
overrides for a node that is already consuming matching label overrides, as
defining a host override would make those label overrides no longer apply.
2. Daemonsets are generated regardless of the current state of the environment.
Ex: If your environment consists of a single node that matches a host override,
the chart will still generate a default daemonset which would fail to schedule
in this example. Likewise if the host or label in the override return no
candidates, these would also fail to schedule.
2. In the event of label conflicts, the last applicable label override defined
takes precedence. In this example, overrides defined for "another_label" would
take precedence and be applied to nodes that contained both of the defined
labels.
Recorded Demo
-------------

View File

@ -63,6 +63,7 @@
{{- end }}
{{- if eq $type "labels" }}
{{- set $.Values "__label_list" . }}
{{- range $label_data := . }}
# dictionary that will contain all info needed to generate this
# iteration of the daemonset.
@ -86,8 +87,10 @@
{{- $list_aggregate := list $label_dict }}
{{- set $.Values.__current_label "matchExpressions" $list_aggregate }}
# Do not schedule to any other specified labels
{{- $other_labels := without $type_data $label_data }}
# Do not schedule to other specified labels, with higher
# precedence as the list position increases. Last defined label
# is highest priority.
{{- $other_labels := without $.Values.__label_list $label_data }}
{{- range $label_data2 := $other_labels }}
{{- $label_dict := omit $label_data2.label "NULL" }}
@ -96,6 +99,7 @@
{{- $list_aggregate := append $.Values.__current_label.matchExpressions $label_dict }}
{{- set $.Values.__current_label "matchExpressions" $list_aggregate }}
{{- end }}
{{- set $.Values "__label_list" $other_labels }}
# Do not schedule to any other specified hosts
{{- range $type, $type_data := $val }}

View File

@ -159,7 +159,8 @@ get_container_status(){
echo "${CLOGS}"
exit 1
fi
elif [ "${status}" = 'INFO Putting the daemon to sleep.' ]; then
elif [ "${status}" = 'INFO Putting the daemon to sleep.' ] ||
[ "${status}" = 'DEBUG + exit 0' ]; then
if [ "${2}" = 'expect_failure' ]; then
echo 'Expected pod to die with error, but pod completed successfully'
echo 'pod logs:'
@ -580,24 +581,15 @@ test_overrides(){
# TODO: Implement more robust tests that do not depend on match expression
# ordering.
# Verify generated affinity for one of the daemonset labels
# Verify generated affinity for test_label
echo "${tc_output}" | grep ' spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: another_label
operator: In
values:
- "another_value"
- key: compute_type
operator: NotIn
values:
- "dpdk"
- "sriov"
- key: test_label
operator: NotIn
operator: In
values:
- "test_value"
- key: kubernetes.io/hostname
@ -616,6 +608,73 @@ test_overrides(){
echo '[SUCCESS] overrides test 2 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 2 failed' && exit 1)
# Verify generated affinity for another_label
echo "${tc_output}" | grep ' spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: another_label
operator: In
values:
- "another_value"
- key: test_label
operator: NotIn
values:
- "test_value"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "superhost"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "helm1"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "specialhost"
hostNetwork: true' &&
echo '[SUCCESS] overrides test 3 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 3 failed' && exit 1)
# Verify generated affinity for compute_type
echo "${tc_output}" | grep ' spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: compute_type
operator: In
values:
- "dpdk"
- "sriov"
- key: another_label
operator: NotIn
values:
- "another_value"
- key: test_label
operator: NotIn
values:
- "test_value"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "superhost"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "helm1"
- key: kubernetes.io/hostname
operator: NotIn
values:
- "specialhost"
hostNetwork: true' &&
echo '[SUCCESS] overrides test 4 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 4 failed' && exit 1)
# Verify generated affinity for one of the daemonset hosts
echo "${tc_output}" | grep ' spec:
affinity:
@ -629,8 +688,8 @@ test_overrides(){
- "soup"
- "chips"
hostNetwork: true' &&
echo '[SUCCESS] overrides test 3 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 3 failed' && exit 1)
echo '[SUCCESS] overrides test 5 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 5 failed' && exit 1)
# Verify generated affinity for one of the daemonset defaults
echo "${tc_output}" | grep ' spec:
@ -665,8 +724,8 @@ test_overrides(){
values:
- "test_value"
hostNetwork: true' &&
echo '[SUCCESS] overrides test 4 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 4 failed' && exit 1)
echo '[SUCCESS] overrides test 6 passed successfully' >> "${TEST_RESULTS}" ||
(echo '[FAILURE] overrides test 6 failed' && exit 1)
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-functional.yaml
key1_override_val=0
@ -686,7 +745,7 @@ test_overrides(){
get_container_status sysctl
_test_sysctl_default $SYSCTL_KEY1 $key1_override_val
_test_sysctl_default $SYSCTL_KEY2 $key2_non_override_val
echo '[SUCCESS] overrides test 5 passed successfully' >> "${TEST_RESULTS}"
echo '[SUCCESS] overrides test 7 passed successfully' >> "${TEST_RESULTS}"
}