From 5b2a7a2439b48ffd7d73c4a828888d7ea4308d19 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Wed, 26 Dec 2018 12:02:21 -0600 Subject: [PATCH] syslog listener for physical nodes - Add a new pod running syslog to receive syslog messages containing the console logs of bootstrapping nodes. This aids in troubleshooting without requiring accessing the OOB console. - Add a UDP forwarder to the MAAS ingress controller as nodes attempt to send syslogs to UDP 514 of the region controller Change-Id: I3f508225f4394a90c6f2534a51f262b42c1afa4e --- .../maas/templates/bin/_start-syslog.sh.tpl | 31 +++++ charts/maas/templates/configmap-bin.yaml | 2 + charts/maas/templates/configmap-etc.yaml | 6 + charts/maas/templates/configmap-ingress.yaml | 1 + charts/maas/templates/etc/_logrotate.conf.tpl | 13 ++ charts/maas/templates/etc/_logrotate.cron.tpl | 1 + charts/maas/templates/etc/_rsyslog.conf.tpl | 11 ++ charts/maas/templates/service-region.yaml | 3 + charts/maas/templates/service-syslog.yaml | 31 +++++ .../templates/statefulset-maas-syslog.yaml | 129 ++++++++++++++++++ charts/maas/templates/statefulset-rack.yaml | 11 +- charts/maas/templates/statefulset-region.yaml | 44 +++++- charts/maas/values.yaml | 55 +++++++- 13 files changed, 333 insertions(+), 5 deletions(-) create mode 100644 charts/maas/templates/bin/_start-syslog.sh.tpl create mode 100644 charts/maas/templates/etc/_logrotate.conf.tpl create mode 100644 charts/maas/templates/etc/_logrotate.cron.tpl create mode 100644 charts/maas/templates/etc/_rsyslog.conf.tpl create mode 100644 charts/maas/templates/service-syslog.yaml create mode 100644 charts/maas/templates/statefulset-maas-syslog.yaml diff --git a/charts/maas/templates/bin/_start-syslog.sh.tpl b/charts/maas/templates/bin/_start-syslog.sh.tpl new file mode 100644 index 0000000..924d9be --- /dev/null +++ b/charts/maas/templates/bin/_start-syslog.sh.tpl @@ -0,0 +1,31 @@ +#!/bin/bash +{{/* + Copyright 2019 AT&T Intellectual Property. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.*/}} +set -x + +RSYSLOG_BIN=${RSYSLOG_BIN:-"/usr/sbin/rsyslogd"} +RSYSLOG_CONFFILE=${RSYSLOG_CONFFILE:-"/etc/rsyslog.conf"} +LOGFILE=${LOGFILE:-"/var/log/maas/nodeboot.log"} + +$RSYSLOG_BIN -f "$RSYSLOG_CONFFILE" + +# Handle race waiting for rsyslogd to start logging +while true +do + tail -f "$LOGFILE" + echo "Waiting for log file to exist." + sleep 10 +done + diff --git a/charts/maas/templates/configmap-bin.yaml b/charts/maas/templates/configmap-bin.yaml index 2f9e43f..43d7bd7 100644 --- a/charts/maas/templates/configmap-bin.yaml +++ b/charts/maas/templates/configmap-bin.yaml @@ -47,3 +47,5 @@ data: {{ tuple "bin/_maas-ingress-errors.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} maas-vip-configure: | {{ tuple "bin/_maas-vip-configure.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + start-syslog.sh: | +{{ tuple "bin/_start-syslog.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/maas/templates/configmap-etc.yaml b/charts/maas/templates/configmap-etc.yaml index d24a9b9..60f12bc 100644 --- a/charts/maas/templates/configmap-etc.yaml +++ b/charts/maas/templates/configmap-etc.yaml @@ -46,3 +46,9 @@ data: {{- end }} sleep-inittab: | ::sysinit:/tmp/maas-vip-configure.sh sleep + rsyslog.conf: | +{{ tuple "etc/_rsyslog.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + logrotate.conf: | +{{ tuple "etc/_logrotate.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + logrotate.cron: | +{{ tuple "etc/_logrotate.cron.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/charts/maas/templates/configmap-ingress.yaml b/charts/maas/templates/configmap-ingress.yaml index e07b9c9..b8bb498 100644 --- a/charts/maas/templates/configmap-ingress.yaml +++ b/charts/maas/templates/configmap-ingress.yaml @@ -31,6 +31,7 @@ metadata: name: maas-ingress-services-udp data: '53': "{{- .Release.Namespace -}}/{{- tuple "maas_region" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" -}}:region-dns" + "514": "{{- .Release.Namespace -}}/{{- tuple "maas_syslog" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" -}}:syslog" ... --- apiVersion: v1 diff --git a/charts/maas/templates/etc/_logrotate.conf.tpl b/charts/maas/templates/etc/_logrotate.conf.tpl new file mode 100644 index 0000000..35cadfb --- /dev/null +++ b/charts/maas/templates/etc/_logrotate.conf.tpl @@ -0,0 +1,13 @@ +{{ printf "%s/%s" .Values.conf.syslog.logpath .Values.conf.syslog.logfile }} +{ + rotate {{ .Values.conf.syslog.logrotate.rotate }} + size {{ .Values.conf.syslog.logrotate.size }} + missingok + delaycompress + compress + nocreate + nomail + postrotate + killall -s HUP rsyslogd >/dev/null 2>&1 || true + endscript +} diff --git a/charts/maas/templates/etc/_logrotate.cron.tpl b/charts/maas/templates/etc/_logrotate.cron.tpl new file mode 100644 index 0000000..f18e083 --- /dev/null +++ b/charts/maas/templates/etc/_logrotate.cron.tpl @@ -0,0 +1 @@ +@hourly root /usr/sbin/logrotate /etc/logrotate.conf diff --git a/charts/maas/templates/etc/_rsyslog.conf.tpl b/charts/maas/templates/etc/_rsyslog.conf.tpl new file mode 100644 index 0000000..6affd3b --- /dev/null +++ b/charts/maas/templates/etc/_rsyslog.conf.tpl @@ -0,0 +1,11 @@ +# Enable the udp server for installation logging +$ModLoad imudp +$UDPServerRun {{ tuple "maas_region" "podport" "region_syslog" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +#$ModLoad imtcp # load TCP listener + +# Reduce message repetition +$RepeatedMsgReduction on +##$RepeatedMsgContainsOriginalMsg on + +:fromhost-ip, !isequal, "127.0.0.1" {{ .Values.conf.syslog.logpath }}/{{ .Values.conf.syslog.logfile }} +& ~ diff --git a/charts/maas/templates/service-region.yaml b/charts/maas/templates/service-region.yaml index 85a2c27..72cc542 100644 --- a/charts/maas/templates/service-region.yaml +++ b/charts/maas/templates/service-region.yaml @@ -38,6 +38,9 @@ spec: port: 53 targetPort: 53 protocol: "UDP" + - name: region-syslog + port: 514 + targetPort: {{ tuple "maas_region" "podport" "region_syslog" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} selector: {{ tuple $envAll "maas" "region" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} {{ if or .Values.network.region_proxy.node_port.enabled .Values.network.region_api.node_port.enabled }} diff --git a/charts/maas/templates/service-syslog.yaml b/charts/maas/templates/service-syslog.yaml new file mode 100644 index 0000000..6909fbd --- /dev/null +++ b/charts/maas/templates/service-syslog.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.syslog_service -}} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "maas_syslog" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: syslog + port: 514 + protocol: "UDP" + targetPort: {{ tuple "maas_syslog" "podport" "syslog" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "maas" "syslog" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +... +{{- end -}} diff --git a/charts/maas/templates/statefulset-maas-syslog.yaml b/charts/maas/templates/statefulset-maas-syslog.yaml new file mode 100644 index 0000000..90ad59c --- /dev/null +++ b/charts/maas/templates/statefulset-maas-syslog.yaml @@ -0,0 +1,129 @@ +{{/* +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. */}} +{{- if .Values.manifests.syslog_statefulset }} +{{- $envAll := . }} +{{- $serviceAccountName := "maas-syslog" }} +{{- $mounts_maas_syslog := .Values.pod.mounts.maas_syslog.syslog }} +{{- $mounts_maas_syslog_init := .Values.pod.mounts.maas_syslog.init_container }} + +{{ tuple $envAll "syslog" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: maas-syslog + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "maas" "syslog" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.syslog }} + podManagementPolicy: 'Parallel' + updateStrategy: + type: 'RollingUpdate' + selector: + matchLabels: +{{ tuple $envAll "maas" "syslog" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "maas" "syslog" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "maas-syslog" "containerNames" (list "syslog") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + shareProcessNamespace: true + affinity: +{{ tuple $envAll "maas" "syslog" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.syslog.node_selector_key }}: {{ .Values.labels.syslog.node_selector_value }} + initContainers: +{{ tuple $envAll "maas_syslog" $mounts_maas_syslog_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: syslog + image: {{ .Values.images.tags.maas_syslog }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/start-syslog.sh + env: + - name: RSYSLOG_CONFFILE + value: "/etc/rsyslog.conf" + - name: LOGFILE + value: {{ printf "%s/%s" .Values.conf.syslog.logpath .Values.conf.syslog.logfile | quote }} + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: /etc/rsyslog.conf + name: maas-etc + subPath: rsyslog.conf + readOnly: true + - mountPath: /tmp/start-syslog.sh + name: maas-bin + subPath: start-syslog.sh + readOnly: true + - mountPath: {{ .Values.conf.syslog.logpath }} + name: syslog-archive + readOnly: false + - name: logrotate + image: {{ .Values.images.tags.maas_syslog }} + imagePullPolicy: {{ .Values.images.pull_policy }} + # Run cron in the foreground and only log failed cronjobs (when logrotate fails) + command: + - "cron" + - "-f" + - "-L" + - "4" + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: /etc/logrotate.conf + name: maas-etc + subPath: logrotate.conf + readOnly: true + - mountPath: /etc/crontab + name: maas-etc + subPath: logrotate.cron + readOnly: true + - mountPath: /var/lib/logrotate + name: logrotate-state + readOnly: false + - mountPath: {{ .Values.conf.syslog.logpath }} + name: syslog-archive + readOnly: false + volumes: + - name: pod-tmp + emptyDir: {} + - name: logrotate-state + emptyDir: {} + - name: maas-etc + configMap: + name: maas-etc + defaultMode: 0444 + - name: maas-bin + configMap: + name: maas-bin + defaultMode: 0555 + volumeClaimTemplates: + - metadata: + name: syslog-archive + annotations: + {{ .Values.storage.syslog.pvc.class_path }}: {{ .Values.storage.syslog.pvc.class_name }} + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.storage.syslog.pvc.size }} +... +{{- end }} diff --git a/charts/maas/templates/statefulset-rack.yaml b/charts/maas/templates/statefulset-rack.yaml index 0fba95d..581804e 100644 --- a/charts/maas/templates/statefulset-rack.yaml +++ b/charts/maas/templates/statefulset-rack.yaml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if .Values.manifests.rack_deployment }} +{{- if .Values.manifests.rack_statefulset }} {{- $envAll := . }} {{- $serviceAccountName := "maas-rack" }} {{- $mounts_maas_rack := .Values.pod.mounts.maas_rack }} @@ -23,13 +23,18 @@ limitations under the License. {{ tuple $envAll "rack_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: StatefulSet metadata: name: maas-rack annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "maas" "rack" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} spec: + selector: + matchLabels: +{{ tuple $envAll "maas" "rack" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} serviceName: maas-rack replicas: {{ .Values.pod.replicas.rack }} podManagementPolicy: 'Parallel' @@ -90,7 +95,7 @@ spec: - 'SYS_TIME' readinessProbe: initialDelaySeconds: 60 - periodSeconds: 300 + periodSeconds: 60 timeoutSeconds: 60 exec: command: diff --git a/charts/maas/templates/statefulset-region.yaml b/charts/maas/templates/statefulset-region.yaml index c238b30..187b667 100644 --- a/charts/maas/templates/statefulset-region.yaml +++ b/charts/maas/templates/statefulset-region.yaml @@ -20,13 +20,18 @@ {{ tuple $envAll "region_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: StatefulSet metadata: name: maas-region annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "maas" "region" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} spec: + selector: + matchLabels: +{{ tuple $envAll "maas" "region" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} serviceName: maas-region-ui replicas: {{ .Values.pod.replicas.region }} updateStrategy: @@ -53,11 +58,36 @@ spec: - name: maas-cache image: {{ .Values.images.tags.maas_cache }} imagePullPolicy: {{ .Values.images.pull_policy }} +{{- end }} +{{- if .Values.manifests.maas_syslog }} + - name: maas-syslog + image: {{ .Values.images.tags.maas_region }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/start-syslog.sh + env: + - name: RSYSLOG_CONFFILE + value: "/etc/rsyslog.conf" + - name: LOGFILE + value: {{ printf "%s/%s" .Values.conf.syslog.logpath .Values.conf.syslog.logfile | quote }} + volueMounts: + - mountPath: /etc/rsyslog.conf + name: maas-etc + subPath: rsyslog.conf + readOnly: true + - mountPath: /tmp/start-syslog.sh + name: maas-bin + subPath: start-syslog.sh + readOnly: true + - mountPath: {{ .Values.conf.syslog.logpath }} + name: syslog-archive + readOnly: false {{- end }} - name: maas-region image: {{ .Values.images.tags.maas_region }} imagePullPolicy: {{ .Values.images.pull_policy }} tty: true + {{ tuple $envAll $envAll.Values.pod.resources.maas_region | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} ports: - name: region-api @@ -149,5 +179,17 @@ spec: secret: secretName: {{ .Values.secrets.maas_region.name }} {{- if $mounts_maas_region.volumes }}{{ toYaml $mounts_maas_region.volumes | indent 8 }}{{ end }} +{{- if .Values.manifests.maas_syslog }} + volumeClaimTemplates: + - metadata: + name: syslog-archive + annotations: + {{ .Values.storage.syslog.pvc.class_path }}: {{ .Values.storage.syslog.pvc.class_name }} + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.storage.syslog.pvc.size }} +{{- end }} ... {{- end }} diff --git a/charts/maas/values.yaml b/charts/maas/values.yaml index 0e7bf10..baaa8ad 100644 --- a/charts/maas/values.yaml +++ b/charts/maas/values.yaml @@ -20,6 +20,7 @@ dependencies: static: maas_ingress: {} + maas_syslog: {} rack_controller: services: - service: maas_region @@ -76,7 +77,9 @@ dependencies: manifests: region_statefulset: true - rack_deployment: true + rack_statefulset: true + syslog_statefulset: true + syslog_service: true test_maas_api: true secret_ssh_key: false ingress_region: true @@ -96,6 +99,7 @@ images: ingress: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.20.0 ingress_vip: docker.io/busybox:latest error_pages: gcr.io/google_containers/defaultbackend:1.0 + maas_syslog: quay.io/airshipit/maas-region-controller:latest pull_policy: IfNotPresent local_registry: # TODO(portdirect): this chart does not yet support local image cacheing @@ -120,6 +124,9 @@ labels: ingress: node_selector_key: ucp-control-plane node_selector_value: enabled + syslog: + node_selector_key: ucp-control-plane + node_selector_value: enabled network: maas_ingress: @@ -141,6 +148,11 @@ network: enabled: false storage: + syslog: + pvc: + class_path: volume.beta.kubernetes.io/storage-class + class_name: general + size: 1Gi rackd: pvc: class_path: volume.beta.kubernetes.io/storage-class @@ -164,6 +176,14 @@ conf: bootaction_url: null cache: enabled: false + syslog: + logpath: /var/log/maas + logfile: nodeboot.log + logrotate: + # How many rotated logs to keep + rotate: '30' + # Size threshold when a log should rotate + size: '100M' maas: override: append: @@ -254,7 +274,16 @@ pod: maas-rack: localhost/docker-default maas-region: maas-region: localhost/docker-default + maas-syslog: + syslog: localhost/docker-default security_context: + maas-syslog: + pod: + runAsUser: 99 + container: + syslog: + runAsUser: 99 + readOnlyRootFilesystem: true ingress_errors: pod: runAsUser: 99 @@ -282,9 +311,13 @@ pod: maas_ingress: init_container: [] maas_region: + maas_syslog: + init_container: [] + syslog: [] replicas: rack: 2 region: 2 + syslog: 1 resources: enabled: false test: @@ -308,6 +341,13 @@ pod: requests: memory: "128Mi" cpu: "500m" + maas_syslog: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "128Mi" + cpu: "500m" maas_ingress_vip: limits: memory: "128Mi" @@ -389,6 +429,16 @@ endpoints: podport: 10254 status: podport: 18080 + maas_syslog: + hosts: + default: maas-syslog + host_fqdn_override: + default: null + port: + syslog: + public: 514 + podport: 514 + maas_db: auth: admin: @@ -432,6 +482,9 @@ endpoints: # comment them out and let the default rule # podport: 8000 # public: 8000 + region_syslog: + default: 514 + podport: 514 host_fqdn_override: default: null physicalprovisioner: