Support MAAS HA

- Create two replicas of rack and region pods
- Use required anti-affinity between rack pods
- Remove the MAAS ingress controller from the rack pod
  and into dedicated deployment
- Update rack registration script to harvest the systemid
  from the underlying host when available

Change-Id: I41e21b7bb5256d04b37a70fbd2088c617b5d239a
This commit is contained in:
Scott Hussey 2018-10-03 16:52:59 -05:00 committed by Scott Hussey
parent 66eb874e63
commit 1c0485e3e6
7 changed files with 362 additions and 228 deletions

View File

@ -2,23 +2,80 @@
set -x
if [[ -r ~maas/maas_id && -r ~maas/secret ]]
then
echo "Found existing maas_id and secret, assuming already registered."
exit 0
fi
# Path where the host's cloud-init data is mounted
# to source the maas system_id
HOST_MOUNT_PATH=${HOST_MOUNT_PATH:-"/host_cloud-init/"}
echo "register-rack-controller URL: ${MAAS_ENDPOINT}"
unregister_maas_rack() {
sys_id="$1"
echo "Deregister this pod as MAAS rack controller ${sys_id}."
maas login local "$MAAS_ENDPOINT" "$MAAS_API_KEY"
maas local rack-controller delete "$sys_id"
rm -f ~maas/maas_id
rm -f ~maas/secret
}
# register forever
while [ 1 ];
do
register_maas_rack() {
sys_id=${1:-""}
echo "register-rack-controller URL: ${MAAS_ENDPOINT}"
if [[ ! -z "$sys_id" ]]
then
echo "Using provided system id ${sys_id}."
echo "$sys_id" > ~maas/maas_id
fi
# register forever
while [ 1 ];
do
if maas-rack register --url=${MAAS_ENDPOINT} --secret="${MAAS_REGION_SECRET}";
then
echo "Successfully registered with MaaS Region Controller"
break
else
echo "Unable to register with ${MAAS_ENDPOINT}... will try again"
sleep 10
sleep 30
fi;
done;
done;
}
get_host_identity() {
# Check if the underlying host was deployed by MAAS
if [[ -r "${HOST_MOUNT_PATH}/instance-data.json" ]]
then
grep -E 'instance-id' "${HOST_MOUNT_PATH}/instance-data.json" | head -1 | tr -d ' ",' | cut -d: -f 2
else
echo ""
fi
}
get_pod_identity() {
if [[ -r ~maas/maas_id ]]
then
cat ~maas/maas_id
else
echo ""
fi
}
HOST_SYSTEM_ID=$(get_host_identity)
POD_SYSTEM_ID=$(get_pod_identity)
# This Pod state already has a MAAS identity
if [[ ! -z "$POD_SYSTEM_ID" ]]
then
# If the pod maas identity doesn't match the
# host maas identity, unregister the pod identity
# as a rack controller
if [[ "$HOST_SYSTEM_ID" != "$POD_SYSTEM_ID" ]]
then
unregister_maas_rack "$POD_SYSTEM_ID"
register_maas_rack "$HOST_SYTEM_ID"
else
echo "Found existing maas_id, assuming already registered."
fi
exit 0
else
register_maas_rack
fi

View File

@ -33,6 +33,6 @@ metadata:
data:
enable-underscores-in-headers: "true"
bind-address: {{ index $bind_address_cidr 0 | quote }}
diable-ipv6: "true"
disable-ipv6: "true"
...
{{- end }}

View File

@ -0,0 +1,267 @@
{{/*
Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.maas_ingress }}
{{- $envAll := . }}
{{- $serviceAccountName := "maas-ingress" }}
{{- $mounts_maas_ingress := .Values.pod.mounts.maas_ingress }}
{{- $mounts_maas_ingress_init := .Values.pod.mounts.maas_ingress.init_container }}
{{ tuple $envAll "maas_ingress" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: {{ $serviceAccountName }}
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
resources:
- ingresses/status
verbs:
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: {{ $serviceAccountName }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ $serviceAccountName }}
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}
namespace: {{ $envAll.Release.Namespace }}
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: {{ $serviceAccountName }}
namespace: {{ $envAll.Release.Namespace }}
rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
- {{ printf "%s-maas-ingress" .Release.Name | quote }}
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
- create
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: {{ $serviceAccountName }}
namespace: {{ $envAll.Release.Namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ $serviceAccountName }}
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}
namespace: {{ $envAll.Release.Namespace }}
---
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: maas-ingress
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
replicas: {{ .Values.pod.replicas.ingress }}
template:
metadata:
labels:
{{ tuple $envAll "maas" "ingress" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
spec:
serviceAccountName: {{ $serviceAccountName }}
affinity:
{{- tuple $envAll "maas" "ingress" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
nodeSelector:
{{ .Values.labels.ingress.node_selector_key }}: {{ .Values.labels.rack.node_selector_value }}
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
initContainers:
{{ tuple $envAll "maas_ingress" $mounts_maas_ingress_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
- name: maas-ingress-vip-init
image: {{ .Values.images.tags.ingress }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
capabilities:
add:
- 'NET_ADMIN'
- 'SYS_MODULE'
runAsUser: 0
command:
- /tmp/maas-vip-configure.sh
- start
env:
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }}
volumeMounts:
- mountPath: /tmp/maas-vip-configure.sh
name: maas-bin
subPath: maas-vip-configure
readOnly: true
- mountPath: /mnt/host-rootfs
name: host-rootfs
readOnly: true
containers:
- name: maas-ingress-vip
image: {{ .Values.images.tags.ingress }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
capabilities:
add:
- 'NET_ADMIN'
runAsUser: 0
command:
- /tmp/maas-vip-configure.sh
- sleep
env:
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }}
volumeMounts:
- mountPath: /tmp/maas-vip-configure.sh
name: maas-bin
subPath: maas-vip-configure
readOnly: true
lifecycle:
preStop:
exec:
command:
- /tmp/maas-vip-configure.sh
- stop
- name: maas-ingress
image: {{ .Values.images.tags.ingress }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
capabilities:
add:
- 'NET_BIND_SERVICE'
runAsUser: 0
command:
- /tmp/maas-ingress.sh
- start
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: RELEASE_NAME
value: {{ .Release.Name | quote }}
- name: HTTP_PORT
value: "8808"
- name: HTTPS_PORT
value: "8543"
- name: HEALTHZ_PORT
value: {{ tuple "maas_ingress" "podport" "healthz" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: STATUS_PORT
value: {{ tuple "maas_ingress" "podport" "status" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: ERROR_PAGE_SERVICE
value: {{ tuple "maas_ingress" "error_pages" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
volumeMounts:
- mountPath: /tmp/maas-ingress.sh
name: maas-bin
subPath: maas-ingress
readOnly: true
{{ if $mounts_maas_ingress.volumeMounts }}{{ toYaml $mounts_maas_ingress.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: maas-bin
configMap:
name: maas-bin
defaultMode: 0555
- name: host-rootfs
hostPath:
path: /
{{ if $mounts_maas_ingress.volumes }}{{ toYaml $mounts_maas_ingress.volumes | indent 8 }}{{ end }}
{{- end }}

View File

@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.maas_ingress }}
---
apiVersion: v1
kind: Service
@ -26,3 +27,4 @@ spec:
selector:
{{ tuple . "maas" "ingress-errors" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
...
{{- end }}

View File

@ -23,125 +23,6 @@ limitations under the License.
{{ tuple $envAll "rack_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: {{ $serviceAccountName }}
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
resources:
- ingresses/status
verbs:
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: {{ $serviceAccountName }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ $serviceAccountName }}
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}
namespace: {{ $envAll.Release.Namespace }}
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: {{ $serviceAccountName }}
namespace: {{ $envAll.Release.Namespace }}
rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
- {{ printf "%s-maas-ingress" .Release.Name | quote }}
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
- create
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: {{ $serviceAccountName }}
namespace: {{ $envAll.Release.Namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ $serviceAccountName }}
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}
namespace: {{ $envAll.Release.Namespace }}
---
---
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
@ -151,6 +32,7 @@ metadata:
spec:
serviceName: maas-rack
replicas: {{ .Values.pod.replicas.rack }}
podManagementPolicy: 'Parallel'
updateStrategy:
type: 'RollingUpdate'
template:
@ -170,93 +52,7 @@ spec:
dnsPolicy: ClusterFirstWithHostNet
initContainers:
{{ tuple $envAll "rack_controller" $mounts_maas_rack_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
- name: maas-ingress-vip-init
image: {{ .Values.images.tags.ingress }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
capabilities:
add:
- 'NET_ADMIN'
- 'SYS_MODULE'
runAsUser: 0
command:
- /tmp/maas-vip-configure.sh
- start
env:
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }}
volumeMounts:
- mountPath: /tmp/maas-vip-configure.sh
name: maas-bin
subPath: maas-vip-configure
readOnly: true
- mountPath: /mnt/host-rootfs
name: host-rootfs
readOnly: true
containers:
- name: maas-ingress-vip
image: {{ .Values.images.tags.ingress }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
capabilities:
add:
- 'NET_ADMIN'
runAsUser: 0
command:
- /tmp/maas-vip-configure.sh
- sleep
env:
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }}
volumeMounts:
- mountPath: /tmp/maas-vip-configure.sh
name: maas-bin
subPath: maas-vip-configure
readOnly: true
lifecycle:
preStop:
exec:
command:
- /tmp/maas-vip-configure.sh
- stop
- name: maas-ingress
image: {{ .Values.images.tags.ingress }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
capabilities:
add:
- 'NET_BIND_SERVICE'
runAsUser: 0
command:
- /tmp/maas-ingress.sh
- start
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: RELEASE_NAME
value: {{ .Release.Name | quote }}
- name: HTTP_PORT
value: "8808"
- name: HTTPS_PORT
value: "8543"
- name: HEALTHZ_PORT
value: {{ tuple "maas_ingress" "podport" "healthz" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: STATUS_PORT
value: {{ tuple "maas_ingress" "podport" "status" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: ERROR_PAGE_SERVICE
value: {{ tuple "maas_ingress" "error_pages" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
volumeMounts:
- mountPath: /tmp/maas-ingress.sh
name: maas-bin
subPath: maas-ingress
readOnly: true
- name: maas-rack
image: {{ .Values.images.tags.maas_rack }}
imagePullPolicy: {{ .Values.images.pull_policy }}
@ -269,6 +65,11 @@ spec:
secretKeyRef:
name: {{ .Values.secrets.maas_region.name }}
key: REGION_SECRET
- name: MAAS_API_KEY
valueFrom:
secretKeyRef:
name: {{ .Values.conf.maas.credentials.secret.name }}
key: 'token'
{{ tuple $envAll $envAll.Values.pod.resources.maas_rack | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command:
- /tmp/start.sh
@ -284,6 +85,9 @@ spec:
name: pod-run-lock
- mountPath: /tmp
name: pod-tmp
- mountPath: /host_cloud-init
name: host-cloud-init
readOnly: true
{{- if .Values.conf.maas.ntp.disable_ntpd_rack }}
- name: maas-bin
mountPath: /usr/sbin/ntpd
@ -324,9 +128,9 @@ spec:
- name: host-sys-fs-cgroup
hostPath:
path: /sys/fs/cgroup
- name: host-rootfs
- name: host-cloud-init
hostPath:
path: /
path: /run/cloud-init
- name: pod-run
emptyDir: {}
- name: pod-run-lock

View File

@ -19,10 +19,13 @@
dependencies:
static:
maas_ingress: {}
rack_controller:
services:
- service: maas_region
endpoint: internal
jobs:
- maas-export-api-key
region_controller:
jobs:
- maas-db-sync
@ -64,6 +67,7 @@ manifests:
secret_ssh_key: false
ingress_region: true
configmap_ingress: true
maas_ingress: true
images:
tags:
@ -98,6 +102,9 @@ labels:
region:
node_selector_key: ucp-control-plane
node_selector_value: enabled
ingress:
node_selector_key: ucp-control-plane
node_selector_value: enabled
network:
maas_ingress:
@ -226,6 +233,7 @@ pod:
affinity:
anti:
type:
rack: requiredDuringSchedulingIgnoredDuringExecution
default: preferredDuringSchedulingIgnoredDuringExecution
topologyKey:
default: kubernetes.io/hostname
@ -239,9 +247,12 @@ pod:
export_api_key:
init_container: []
export_api_key:
maas_ingress:
init_container: []
maas_region:
replicas:
rack: 1
region: 1
rack: 2
region: 2
resources:
enabled: false
test:

View File

@ -40,13 +40,6 @@ ENV MAAS_VERSION 2.3.5-6511-gf466fdb-0ubuntu1
# install maas
RUN rsyslogd; apt-get install -y maas-cli=$MAAS_VERSION maas-rack-controller=$MAAS_VERSION
COPY scripts/register-rack-controller.sh /usr/local/bin
RUN chmod +x /usr/local/bin/register-rack-controller.sh
# register ourselves with the region controller
COPY scripts/register-rack-controller.service /lib/systemd/system/register-rack-controller.service
RUN systemctl enable register-rack-controller.service
RUN mv /usr/sbin/tcpdump /usr/bin/tcpdump
RUN ln -s /usr/bin/tcpdump /usr/sbin/tcpdump