diff --git a/charts/maas/templates/bin/_register-rack-controller.sh.tpl b/charts/maas/templates/bin/_register-rack-controller.sh.tpl index 85256e9..3280643 100644 --- a/charts/maas/templates/bin/_register-rack-controller.sh.tpl +++ b/charts/maas/templates/bin/_register-rack-controller.sh.tpl @@ -2,23 +2,80 @@ set -x -if [[ -r ~maas/maas_id && -r ~maas/secret ]] -then - echo "Found existing maas_id and secret, assuming already registered." - exit 0 -fi +# Path where the host's cloud-init data is mounted +# to source the maas system_id +HOST_MOUNT_PATH=${HOST_MOUNT_PATH:-"/host_cloud-init/"} -echo "register-rack-controller URL: ${MAAS_ENDPOINT}" +unregister_maas_rack() { + sys_id="$1" + echo "Deregister this pod as MAAS rack controller ${sys_id}." + maas login local "$MAAS_ENDPOINT" "$MAAS_API_KEY" + maas local rack-controller delete "$sys_id" + rm -f ~maas/maas_id + rm -f ~maas/secret +} -# register forever -while [ 1 ]; -do +register_maas_rack() { + sys_id=${1:-""} + echo "register-rack-controller URL: ${MAAS_ENDPOINT}" + + if [[ ! -z "$sys_id" ]] + then + echo "Using provided system id ${sys_id}." + echo "$sys_id" > ~maas/maas_id + fi + + # register forever + while [ 1 ]; + do if maas-rack register --url=${MAAS_ENDPOINT} --secret="${MAAS_REGION_SECRET}"; then echo "Successfully registered with MaaS Region Controller" break else echo "Unable to register with ${MAAS_ENDPOINT}... will try again" - sleep 10 + sleep 30 fi; -done; + done; +} + +get_host_identity() { + # Check if the underlying host was deployed by MAAS + if [[ -r "${HOST_MOUNT_PATH}/instance-data.json" ]] + then + grep -E 'instance-id' "${HOST_MOUNT_PATH}/instance-data.json" | head -1 | tr -d ' ",' | cut -d: -f 2 + else + echo "" + fi +} + +get_pod_identity() { + if [[ -r ~maas/maas_id ]] + then + cat ~maas/maas_id + else + echo "" + fi +} + +HOST_SYSTEM_ID=$(get_host_identity) +POD_SYSTEM_ID=$(get_pod_identity) + +# This Pod state already has a MAAS identity +if [[ ! -z "$POD_SYSTEM_ID" ]] +then + # If the pod maas identity doesn't match the + # host maas identity, unregister the pod identity + # as a rack controller + if [[ "$HOST_SYSTEM_ID" != "$POD_SYSTEM_ID" ]] + then + unregister_maas_rack "$POD_SYSTEM_ID" + register_maas_rack "$HOST_SYTEM_ID" + else + echo "Found existing maas_id, assuming already registered." + fi + + exit 0 +else + register_maas_rack +fi diff --git a/charts/maas/templates/configmap-ingress.yaml b/charts/maas/templates/configmap-ingress.yaml index e289e27..d2dde34 100644 --- a/charts/maas/templates/configmap-ingress.yaml +++ b/charts/maas/templates/configmap-ingress.yaml @@ -33,6 +33,6 @@ metadata: data: enable-underscores-in-headers: "true" bind-address: {{ index $bind_address_cidr 0 | quote }} - diable-ipv6: "true" + disable-ipv6: "true" ... {{- end }} diff --git a/charts/maas/templates/deployment-maas-ingress.yaml b/charts/maas/templates/deployment-maas-ingress.yaml new file mode 100644 index 0000000..a46e597 --- /dev/null +++ b/charts/maas/templates/deployment-maas-ingress.yaml @@ -0,0 +1,267 @@ + +{{/* +Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.maas_ingress }} +{{- $envAll := . }} +{{- $serviceAccountName := "maas-ingress" }} +{{- $mounts_maas_ingress := .Values.pod.mounts.maas_ingress }} +{{- $mounts_maas_ingress_init := .Values.pod.mounts.maas_ingress.init_container }} + +{{ tuple $envAll "maas_ingress" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + resources: + - ingresses/status + verbs: + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - {{ printf "%s-maas-ingress" .Release.Name | quote }} + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: maas-ingress + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + replicas: {{ .Values.pod.replicas.ingress }} + template: + metadata: + labels: +{{ tuple $envAll "maas" "ingress" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{- tuple $envAll "maas" "ingress" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.ingress.node_selector_key }}: {{ .Values.labels.rack.node_selector_value }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + initContainers: +{{ tuple $envAll "maas_ingress" $mounts_maas_ingress_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: maas-ingress-vip-init + image: {{ .Values.images.tags.ingress }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + capabilities: + add: + - 'NET_ADMIN' + - 'SYS_MODULE' + runAsUser: 0 + command: + - /tmp/maas-vip-configure.sh + - start + env: +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }} + volumeMounts: + - mountPath: /tmp/maas-vip-configure.sh + name: maas-bin + subPath: maas-vip-configure + readOnly: true + - mountPath: /mnt/host-rootfs + name: host-rootfs + readOnly: true + containers: + - name: maas-ingress-vip + image: {{ .Values.images.tags.ingress }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + capabilities: + add: + - 'NET_ADMIN' + runAsUser: 0 + command: + - /tmp/maas-vip-configure.sh + - sleep + env: +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }} + volumeMounts: + - mountPath: /tmp/maas-vip-configure.sh + name: maas-bin + subPath: maas-vip-configure + readOnly: true + lifecycle: + preStop: + exec: + command: + - /tmp/maas-vip-configure.sh + - stop + - name: maas-ingress + image: {{ .Values.images.tags.ingress }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + capabilities: + add: + - 'NET_BIND_SERVICE' + runAsUser: 0 + command: + - /tmp/maas-ingress.sh + - start + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: RELEASE_NAME + value: {{ .Release.Name | quote }} + - name: HTTP_PORT + value: "8808" + - name: HTTPS_PORT + value: "8543" + - name: HEALTHZ_PORT + value: {{ tuple "maas_ingress" "podport" "healthz" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: STATUS_PORT + value: {{ tuple "maas_ingress" "podport" "status" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ERROR_PAGE_SERVICE + value: {{ tuple "maas_ingress" "error_pages" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }} + volumeMounts: + - mountPath: /tmp/maas-ingress.sh + name: maas-bin + subPath: maas-ingress + readOnly: true +{{ if $mounts_maas_ingress.volumeMounts }}{{ toYaml $mounts_maas_ingress.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: maas-bin + configMap: + name: maas-bin + defaultMode: 0555 + - name: host-rootfs + hostPath: + path: / +{{ if $mounts_maas_ingress.volumes }}{{ toYaml $mounts_maas_ingress.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/charts/maas/templates/service-ingress-error.yaml b/charts/maas/templates/service-ingress-error.yaml index 4ff0712..fd9bfb0 100644 --- a/charts/maas/templates/service-ingress-error.yaml +++ b/charts/maas/templates/service-ingress-error.yaml @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */}} +{{- if .Values.manifests.maas_ingress }} --- apiVersion: v1 kind: Service @@ -26,3 +27,4 @@ spec: selector: {{ tuple . "maas" "ingress-errors" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} ... +{{- end }} diff --git a/charts/maas/templates/statefulset-rack.yaml b/charts/maas/templates/statefulset-rack.yaml index 6b3832f..47a5cba 100644 --- a/charts/maas/templates/statefulset-rack.yaml +++ b/charts/maas/templates/statefulset-rack.yaml @@ -23,125 +23,6 @@ limitations under the License. {{ tuple $envAll "rack_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: {{ $serviceAccountName }} -rules: - - apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: {{ $serviceAccountName }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ $serviceAccountName }} -subjects: - - kind: ServiceAccount - name: {{ $serviceAccountName }} - namespace: {{ $envAll.Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: {{ $serviceAccountName }} - namespace: {{ $envAll.Release.Namespace }} -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - - {{ printf "%s-maas-ingress" .Release.Name | quote }} - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - create - - update ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: {{ $serviceAccountName }} - namespace: {{ $envAll.Release.Namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ $serviceAccountName }} -subjects: - - kind: ServiceAccount - name: {{ $serviceAccountName }} - namespace: {{ $envAll.Release.Namespace }} ---- ---- apiVersion: apps/v1beta1 kind: StatefulSet metadata: @@ -151,6 +32,7 @@ metadata: spec: serviceName: maas-rack replicas: {{ .Values.pod.replicas.rack }} + podManagementPolicy: 'Parallel' updateStrategy: type: 'RollingUpdate' template: @@ -170,93 +52,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet initContainers: {{ tuple $envAll "rack_controller" $mounts_maas_rack_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - - name: maas-ingress-vip-init - image: {{ .Values.images.tags.ingress }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - securityContext: - capabilities: - add: - - 'NET_ADMIN' - - 'SYS_MODULE' - runAsUser: 0 - command: - - /tmp/maas-vip-configure.sh - - start - env: -{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }} - volumeMounts: - - mountPath: /tmp/maas-vip-configure.sh - name: maas-bin - subPath: maas-vip-configure - readOnly: true - - mountPath: /mnt/host-rootfs - name: host-rootfs - readOnly: true containers: - - name: maas-ingress-vip - image: {{ .Values.images.tags.ingress }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress_vip | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - securityContext: - capabilities: - add: - - 'NET_ADMIN' - runAsUser: 0 - command: - - /tmp/maas-vip-configure.sh - - sleep - env: -{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.network.maas_ingress | indent 12 }} - volumeMounts: - - mountPath: /tmp/maas-vip-configure.sh - name: maas-bin - subPath: maas-vip-configure - readOnly: true - lifecycle: - preStop: - exec: - command: - - /tmp/maas-vip-configure.sh - - stop - - name: maas-ingress - image: {{ .Values.images.tags.ingress }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.maas_ingress | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - securityContext: - capabilities: - add: - - 'NET_BIND_SERVICE' - runAsUser: 0 - command: - - /tmp/maas-ingress.sh - - start - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: RELEASE_NAME - value: {{ .Release.Name | quote }} - - name: HTTP_PORT - value: "8808" - - name: HTTPS_PORT - value: "8543" - - name: HEALTHZ_PORT - value: {{ tuple "maas_ingress" "podport" "healthz" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - - name: STATUS_PORT - value: {{ tuple "maas_ingress" "podport" "status" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - - name: ERROR_PAGE_SERVICE - value: {{ tuple "maas_ingress" "error_pages" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }} - volumeMounts: - - mountPath: /tmp/maas-ingress.sh - name: maas-bin - subPath: maas-ingress - readOnly: true - name: maas-rack image: {{ .Values.images.tags.maas_rack }} imagePullPolicy: {{ .Values.images.pull_policy }} @@ -269,6 +65,11 @@ spec: secretKeyRef: name: {{ .Values.secrets.maas_region.name }} key: REGION_SECRET + - name: MAAS_API_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.conf.maas.credentials.secret.name }} + key: 'token' {{ tuple $envAll $envAll.Values.pod.resources.maas_rack | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} command: - /tmp/start.sh @@ -284,6 +85,9 @@ spec: name: pod-run-lock - mountPath: /tmp name: pod-tmp + - mountPath: /host_cloud-init + name: host-cloud-init + readOnly: true {{- if .Values.conf.maas.ntp.disable_ntpd_rack }} - name: maas-bin mountPath: /usr/sbin/ntpd @@ -324,9 +128,9 @@ spec: - name: host-sys-fs-cgroup hostPath: path: /sys/fs/cgroup - - name: host-rootfs + - name: host-cloud-init hostPath: - path: / + path: /run/cloud-init - name: pod-run emptyDir: {} - name: pod-run-lock diff --git a/charts/maas/values.yaml b/charts/maas/values.yaml index 98f72aa..bfcaf2c 100644 --- a/charts/maas/values.yaml +++ b/charts/maas/values.yaml @@ -19,10 +19,13 @@ dependencies: static: + maas_ingress: {} rack_controller: services: - service: maas_region endpoint: internal + jobs: + - maas-export-api-key region_controller: jobs: - maas-db-sync @@ -64,6 +67,7 @@ manifests: secret_ssh_key: false ingress_region: true configmap_ingress: true + maas_ingress: true images: tags: @@ -98,6 +102,9 @@ labels: region: node_selector_key: ucp-control-plane node_selector_value: enabled + ingress: + node_selector_key: ucp-control-plane + node_selector_value: enabled network: maas_ingress: @@ -226,6 +233,7 @@ pod: affinity: anti: type: + rack: requiredDuringSchedulingIgnoredDuringExecution default: preferredDuringSchedulingIgnoredDuringExecution topologyKey: default: kubernetes.io/hostname @@ -239,9 +247,12 @@ pod: export_api_key: init_container: [] export_api_key: + maas_ingress: + init_container: [] + maas_region: replicas: - rack: 1 - region: 1 + rack: 2 + region: 2 resources: enabled: false test: diff --git a/images/maas-rack-controller/Dockerfile b/images/maas-rack-controller/Dockerfile index 73510ac..f99f5a7 100644 --- a/images/maas-rack-controller/Dockerfile +++ b/images/maas-rack-controller/Dockerfile @@ -40,13 +40,6 @@ ENV MAAS_VERSION 2.3.5-6511-gf466fdb-0ubuntu1 # install maas RUN rsyslogd; apt-get install -y maas-cli=$MAAS_VERSION maas-rack-controller=$MAAS_VERSION -COPY scripts/register-rack-controller.sh /usr/local/bin -RUN chmod +x /usr/local/bin/register-rack-controller.sh - -# register ourselves with the region controller -COPY scripts/register-rack-controller.service /lib/systemd/system/register-rack-controller.service -RUN systemctl enable register-rack-controller.service - RUN mv /usr/sbin/tcpdump /usr/bin/tcpdump RUN ln -s /usr/bin/tcpdump /usr/sbin/tcpdump