From b648edfe40dc65a148a45f47e9b3573dc3922756 Mon Sep 17 00:00:00 2001 From: Phil Sphicas Date: Thu, 8 Jul 2021 05:35:42 +0000 Subject: [PATCH] Deploy MAAS 2.8 on Ubuntu bionic Image changes: * base image ubuntu:18.04 * MAAS version 2.8.6-8602-g.07cdffcaa-0ubuntu1~18.04.1 from ppa/2.8 * default contents of /var/lib/maas are archived in /opt/maas * updated patches: - 2.3_bios_grub_partition.patch, changed in maas [0] - 2.3_partitiontable_does_not_exist.patch, changed in maas [1] [2] - 2.3_secure_headers.patch, updated for twisted 17.9.0 [3] * removed patches: - 2.3_bios_grub_preseed.patch, changed in maas, now N/A [0] - 2.3_hostheader.patch, fixed in maas [4] - 2.3_maas_enlist.patch, fixed in maas [5] - 2.3_mac_address.patch, fixed in maas [6] * new patches: - 2.8_maas_ipmi_autodetect_tool.patch, enlistment reliability * reformatted patches due to blackening change [1]: - 2.3_configure_ipmi_user.patch - 2.3_ipmi_error.patch - 2.3_kernel_package.patch, custom req to specify kernel package - 2.3_nic_filter.patch, custom req to ignore cali* interfaces - 2.3_region_secret_rotate.patch - 2.3_route.patch Chart changes: * maas-region podport is 5240 * maas config option http_boot is no longer configurable [7] * start script restores some default files into /var/lib/maas * register-rack-controller script removes old files in /etc/maas * enlist userdata now matches commissioning/curtin userdata [8] * force_gpt option is removed [9], as GPT is now the default * update to configure remote_syslog in import resources job [10] * enlist_commissioning is disabled for backwards compatibility [11] 0: https://github.com/maas/maas/commit/d8e234eb09a59ac0b25033a2346588f7bd87562f 1: https://github.com/maas/maas/commit/db30bb39fa9d98bf75bcd9f7c3d525d5d9e1a21a 2: https://github.com/maas/maas/commit/665feb7575ed474b5dbeb2017c53c9b545a65d35 3: https://github.com/twisted/twisted/blob/twisted-17.9.0/src/twisted/web/server.py 4: https://github.com/maas/maas/commit/573da6972903c1b5566aad3012b97b09108d46eb 5: https://github.com/maas/maas/commit/d390a1da6ac05483f4cdc82c6416fd7c57c22e2a 6: https://github.com/maas/maas/commit/34631c2fe5062ac9078c8175d3fabf7eef394d8a 7: https://github.com/maas/maas/commit/0e94c26a53e0f9caad6dae35cc1fbe352bb9a067 8: https://github.com/maas/maas/commit/22641cffccea44ded1d20a72d84ba23856730d05 9: https://opendev.org/airship/maas/commit/97c25a048689a5e40d97be2cc17e50e8f41d9249 10: https://github.com/maas/maas/commit/d67c359c7b7c300bd3421ee8993b55f5f08e702f 11: https://github.com/maas/maas/commit/51b9712c201c15d0425ed86f015133d4a6bb0cf9 Change-Id: I0685d76cf083ff5aa33c8db552059721289d5c53 --- .zuul.yaml | 2 +- Makefile | 6 +- README.md | 2 +- .../bin/_import-boot-resources.sh.tpl | 7 +- .../bin/_register-rack-controller.sh.tpl | 3 + charts/maas/templates/bin/_start.sh.tpl | 9 +- charts/maas/templates/etc/_enlist.tpl | 11 +-- charts/maas/templates/job-import.yaml | 8 +- charts/maas/values.yaml | 25 +++--- .../maas-rack-controller/2.3_hostheader.patch | 10 --- .../maas-rack-controller/2.3_ipmi_error.patch | 29 ++++--- .../2.3_mac_address.patch | 6 -- .../maas-rack-controller/2.3_nic_filter.patch | 18 ++-- .../2.3_secure_headers.patch | 14 ++- images/maas-rack-controller/Dockerfile | 60 ++++++------- .../2.3_bios_grub_partition.patch | 36 ++++---- .../2.3_bios_grub_preseed.patch | 22 ----- .../2.3_configure_ipmi_user.patch | 18 ++-- .../2.3_kernel_package.patch | 20 +++-- .../2.3_maas_enlist.patch | 13 --- .../2.3_mac_address.patch | 6 -- .../2.3_partitiontable_does_not_exist.patch | 8 +- .../2.3_region_secret_rotate.patch | 13 +-- images/maas-region-controller/2.3_route.patch | 20 +++-- .../2.3_secure_headers.patch | 14 ++- .../2.8_maas_ipmi_autodetect_tool.patch | 17 ++++ images/maas-region-controller/Dockerfile | 87 +++++++------------ images/sstream-cache/Dockerfile | 6 +- 28 files changed, 222 insertions(+), 268 deletions(-) delete mode 100644 images/maas-rack-controller/2.3_hostheader.patch delete mode 100644 images/maas-rack-controller/2.3_mac_address.patch delete mode 100644 images/maas-region-controller/2.3_bios_grub_preseed.patch delete mode 100644 images/maas-region-controller/2.3_maas_enlist.patch delete mode 100644 images/maas-region-controller/2.3_mac_address.patch create mode 100644 images/maas-region-controller/2.8_maas_ipmi_autodetect_tool.patch diff --git a/.zuul.yaml b/.zuul.yaml index b39f4f6..51b6cd0 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -33,7 +33,7 @@ name: airship-maas-single-node nodes: - name: primary - label: ubuntu-xenial + label: ubuntu-bionic - job: name: airship-maas-lint-ws diff --git a/Makefile b/Makefile index 1cdc172..113375c 100644 --- a/Makefile +++ b/Makefile @@ -32,9 +32,9 @@ COMMIT ?= $(shell git rev-parse HEAD) IMAGE_NAME := maas-rack-controller maas-region-controller sstream-cache BUILD_DIR := $(shell mktemp -d) HELM := $(BUILD_DIR)/helm -SSTREAM_IMAGE := "https://images.maas.io/ephemeral-v3/daily/" -SSTREAM_RELEASE := "xenial" -UBUNTU_BASE_IMAGE ?= ubuntu:16.04 +SSTREAM_IMAGE := "https://images.maas.io/ephemeral-v3/stable/" +SSTREAM_RELEASE := "bionic" +UBUNTU_BASE_IMAGE ?= ubuntu:18.04 .PHONY: images #Build all images in the list diff --git a/README.md b/README.md index da383dc..e254a31 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ This image needs to be run in privileged host networking mode to function. The cache image [Dockerfile](images/sstream-cache/Dockerfile) simply provides a point-in-time mirror of the maas.io image repository so that if you are deploying MaaS somewhere without network connectivity, you have a local copy of Ubuntu. Currently this only -mirrors Ubuntu 16.04 Xenial and does not update the mirror after image creation. +mirrors Ubuntu 18.04 Bionic and does not update the mirror after image creation. ## Charts diff --git a/charts/maas/templates/bin/_import-boot-resources.sh.tpl b/charts/maas/templates/bin/_import-boot-resources.sh.tpl index 341f64c..a00797a 100644 --- a/charts/maas/templates/bin/_import-boot-resources.sh.tpl +++ b/charts/maas/templates/bin/_import-boot-resources.sh.tpl @@ -154,6 +154,10 @@ function configure_dns { check_then_set upstream_dns ${MAAS_DNS_SERVERS} } +function configure_syslog { + check_then_set remote_syslog ${MAAS_REMOTE_SYSLOG} +} + function configure_images { check_for_rack_sync @@ -175,8 +179,6 @@ function configure_boot_sources { maas ${ADMIN_USERNAME} boot-source update 1 url=http://localhost:8888/maas/images/ephemeral-v3/daily/ fi - check_then_set http_boot ${MAAS_HTTP_BOOT} - selected_releases="$(maas ${ADMIN_USERNAME} boot-source-selections read 1 | jq -r '.[] | .release')" if ! echo "${selected_releases}" | grep -q "${MAAS_DEFAULT_DISTRO}" @@ -213,6 +215,7 @@ timer "$RETRY_TIMER" maas_login configure_proxy configure_ntp configure_dns +configure_syslog configure_extra_settings # make call to import images diff --git a/charts/maas/templates/bin/_register-rack-controller.sh.tpl b/charts/maas/templates/bin/_register-rack-controller.sh.tpl index bfa9937..265fab9 100644 --- a/charts/maas/templates/bin/_register-rack-controller.sh.tpl +++ b/charts/maas/templates/bin/_register-rack-controller.sh.tpl @@ -9,6 +9,9 @@ unregister_maas_rack() { echo "Deregistering this pod's local state in /var/lib/maas directory." rm -f /var/lib/maas/secret rm -f /var/lib/maas/maas_id + echo "Removing local state in /etc/maas directory." + rm -f /etc/maas/rackd.conf + rm -rf /etc/maas/certificates } register_maas_rack() { diff --git a/charts/maas/templates/bin/_start.sh.tpl b/charts/maas/templates/bin/_start.sh.tpl index f5d4bef..ad2a879 100644 --- a/charts/maas/templates/bin/_start.sh.tpl +++ b/charts/maas/templates/bin/_start.sh.tpl @@ -20,9 +20,13 @@ set -ex env > /tmp/env # Ensure PVC volumes have correct ownership +# Also restore the subdirectory structure and any default files +# (i.e. /var/lib/maas/http/nginx.conf) chown maas:maas ~maas/ chown maas:maas /etc/maas +[[ -r /opt/maas/var-lib-maas.tgz ]] && tar -C/ -xvzf /opt/maas/var-lib-maas.tgz +[[ -d ~maas/boot-resources ]] && chown -R maas:maas ~maas/boot-resources # MAAS must be able to ssh to libvirt hypervisors # to control VMs @@ -52,10 +56,5 @@ done if [[ $sh_set = false ]]; then exit 1 fi -{{- if .Values.conf.maas.force_gpt }} -# Forcing the use of GPT irrespective of boot disk size -# https://github.com/maas/maas/blob/2.3/src/maasserver/models/partitiontable.py#L51-L53 -sed -i '/^GPT_REQUIRED_SIZE =/c\GPT_REQUIRED_SIZE = 0' /usr/lib/python3/dist-packages/maasserver/models/partitiontable.py -{{- end }} set -e exec /sbin/init --log-target=console 3>&1 diff --git a/charts/maas/templates/etc/_enlist.tpl b/charts/maas/templates/etc/_enlist.tpl index 0f71d9b..105d153 100644 --- a/charts/maas/templates/etc/_enlist.tpl +++ b/charts/maas/templates/etc/_enlist.tpl @@ -1,13 +1,4 @@ -#cloud-config -datasource: - MAAS: - timeout : 50 - max_wait : 120 - # there are no default values for metadata_url or oauth credentials - # If no credentials are present, non-authed attempts will be made. - metadata_url: {{ "{{" }}metadata_enlist_url{{ "}}" }} - -output: {all: '| tee -a /var/log/cloud-init-output.log'} +{{ "{{" }}preseed_data{{ "}}" }} {{- range $k, $v := .Values.conf.cloudconfig.sections }} {{ dict $k $v | toYaml | trim }} {{- end }} diff --git a/charts/maas/templates/job-import.yaml b/charts/maas/templates/job-import.yaml index a8709fe..f588d78 100644 --- a/charts/maas/templates/job-import.yaml +++ b/charts/maas/templates/job-import.yaml @@ -72,8 +72,6 @@ spec: value: {{ .Values.conf.maas.proxy.proxy_server }} - name: MAAS_INTERNAL_PROXY_PORT value: {{ tuple "maas_region" "default" "region_proxy" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - - name: MAAS_HTTP_BOOT - value: {{ .Values.conf.maas.http_boot | quote }} - name: MAAS_NTP_SERVERS value: {{ .Values.conf.maas.ntp.ntp_servers | join "," | quote }} - name: MAAS_NTP_EXTERNAL_ONLY @@ -88,6 +86,12 @@ spec: value: {{ .Values.conf.maas.images.default_image | quote }} - name: MAAS_DEFAULT_KERNEL value: {{ .Values.conf.maas.images.default_kernel | quote }} + - name: MAAS_REMOTE_SYSLOG +{{- if empty .Values.conf.maas.syslog.remote_syslog }} + value: {{ tuple "maas_syslog" "public" "syslog" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" | quote }} +{{- else }} + value: {{ .Values.conf.maas.syslog.remote_syslog | quote }} +{{- end }} {{ tuple $envAll $envAll.Values.pod.resources.jobs.import_resources | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ dict "envAll" $envAll "application" "import_resources" "container" "region_import_resources" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} command: diff --git a/charts/maas/values.yaml b/charts/maas/values.yaml index 3ea9abf..2f00a71 100644 --- a/charts/maas/values.yaml +++ b/charts/maas/values.yaml @@ -225,7 +225,6 @@ conf: maas: override: append: - http_boot: true url: maas_url: null ingress_disable_gui: false @@ -253,18 +252,22 @@ conf: proxy_server: null images: default_os: 'ubuntu' - default_image: 'xenial' - default_kernel: 'ga-16.04' + default_image: 'bionic' + default_kernel: 'ga-18.04' credentials: secret: namespace: maas name: maas-api-key - # By default, MAAS will use MBR for boot disks smaller than 2 TiB. - # Set force_gpt: true to always use GPT. - # NOTE: This is not a standard MAAS setting, and enabling it will cause - # modification of a file during maas-region pod startup: - # /usr/lib/python3/dist-packages/maasserver/models/partitiontable.py - force_gpt: false + syslog: + # Remote syslog destination for machine syslogs (during enlistment, + # commissioning, and deployment) + # Can be specified as 'ip', 'fqdn', 'ip:port', or 'fqdn:port', where + # ':port' defaults to ':514' + # If remote_syslog is null, the destination will be resolved via a + # host_and_port_endpoint_uri_lookup of the public maas_syslog endpoint + # (Note that this differs from the MAAS default, which is to send machine + # syslogs to MAAS on port 5247) + remote_syslog: null extra_settings: # Additional settings available via maas $PROFILE maas set-config # Marks if the initial intro has been completed: true or false @@ -275,6 +278,8 @@ conf: network_discovery: disabled # active_discovery_interval (seconds): one of '0', '604800', '86400', '43200', '21600', '10800', '3600', '1800', '600' active_discovery_interval: 0 + # enlist_commissioning: if true, directly go into commissioning during enlistment + enlist_commissioning: false # system user for console login/recovery in early phases of deployment system_user: 'root' system_passwd: 'password' @@ -654,7 +659,7 @@ endpoints: region_api: default: 80 nodeport: 31900 - podport: 80 + podport: 5240 public: 80 region_proxy: default: 8000 diff --git a/images/maas-rack-controller/2.3_hostheader.patch b/images/maas-rack-controller/2.3_hostheader.patch deleted file mode 100644 index 6cd77bd..0000000 --- a/images/maas-rack-controller/2.3_hostheader.patch +++ /dev/null @@ -1,10 +0,0 @@ -1047c1047 -< if family in {AF_INET, AF_INET6}: ---- -> if family in {AF_INET6}: -1051a1052,1054 -> info_url = info_url_base._replace(netloc=netloc) -> elif family in {AF_INET}: -> info_url = info_url_base -1054d1056 -< info_url = info_url_base._replace(netloc=netloc) diff --git a/images/maas-rack-controller/2.3_ipmi_error.patch b/images/maas-rack-controller/2.3_ipmi_error.patch index 703fa5f..6c14876 100644 --- a/images/maas-rack-controller/2.3_ipmi_error.patch +++ b/images/maas-rack-controller/2.3_ipmi_error.patch @@ -1,26 +1,27 @@ diff --git a/src/provisioningserver/drivers/power/ipmi.py b/src/provisioningserver/drivers/power/ipmi.py -index 219ee268b..acdf2065d 100644 +index e99b807ce..8f56dc77a 100644 --- a/src/provisioningserver/drivers/power/ipmi.py +++ b/src/provisioningserver/drivers/power/ipmi.py -@@ -143,6 +143,12 @@ IPMI_ERRORS = { - " MAAS performed several retries. Please wait and try again."), - 'exception': PowerConnError +@@ -154,6 +154,13 @@ IPMI_ERRORS = { + ), + "exception": PowerConnError, }, -+ 'BMC error': { -+ 'message': ( ++ "BMC error": { ++ "message": ( + "Device not responding correctly while performing power action." -+ " MAAS performed several retries. Please wait and try again."), -+ 'exception': PowerConnError ++ " MAAS performed several retries. Please wait and try again." ++ ), ++ "exception": PowerConnError, + }, - 'could not find inband device': { - 'message': ( + "could not find inband device": { + "message": ( "An inband device could not be found." -@@ -209,7 +215,7 @@ class IPMIPowerDriver(PowerDriver): - 'mac_address', "Power MAC", scope=SETTING_SCOPE.NODE) +@@ -231,7 +238,7 @@ class IPMIPowerDriver(PowerDriver): + ), ] - ip_extractor = make_ip_extractor('power_address') + ip_extractor = make_ip_extractor("power_address") - wait_time = (4, 8, 16, 32) + wait_time = (4, 4, 8, 8, 16, 16, 32, 32) def detect_missing_packages(self): - if not shell.has_command_available('ipmipower'): + if not shell.has_command_available("ipmipower"): diff --git a/images/maas-rack-controller/2.3_mac_address.patch b/images/maas-rack-controller/2.3_mac_address.patch deleted file mode 100644 index c709316..0000000 --- a/images/maas-rack-controller/2.3_mac_address.patch +++ /dev/null @@ -1,6 +0,0 @@ -394a395,399 -> # Exclude interfaces that have duplicate MACs -> # such as OVS gretap and erspan interfaces -> ifaces = { k: v -> for k, v in ifaces.items() -> if v.get('mac', '') != '00:00:00:00:00:00'} diff --git a/images/maas-rack-controller/2.3_nic_filter.patch b/images/maas-rack-controller/2.3_nic_filter.patch index cd6e710..fcdebc2 100644 --- a/images/maas-rack-controller/2.3_nic_filter.patch +++ b/images/maas-rack-controller/2.3_nic_filter.patch @@ -1,13 +1,13 @@ diff --git a/src/provisioningserver/utils/network.py b/src/provisioningserver/utils/network.py -index 960d3eb..cd627fd 100644 +index 55f8449f4..ebde045ee 100644 --- a/src/provisioningserver/utils/network.py +++ b/src/provisioningserver/utils/network.py -@@ -1138,7 +1138,7 @@ def get_all_interfaces_definition(annotate_with_monitored: bool=True) -> dict: - interfaces = {} - dhclient_info = get_dhclient_info() - iproute_info = get_ip_route() -- exclude_types = ["loopback", "ipip"] -+ exclude_types = ["loopback", "ipip", "ethernet"] +@@ -1153,6 +1153,8 @@ def get_all_interfaces_definition( + # interfaces for guests. By themselves, they're not useful for MAAS to + # manage. + "ethernet.tunnel", ++ # Always exclude non-specific ethernet interfaces. ++ "ethernet", + ] if not running_in_container(): - exclude_types.append("ethernet") - ipaddr_info = { + # When not running in a container, we should be able to identify diff --git a/images/maas-rack-controller/2.3_secure_headers.patch b/images/maas-rack-controller/2.3_secure_headers.patch index 6c006eb..c7a520e 100644 --- a/images/maas-rack-controller/2.3_secure_headers.patch +++ b/images/maas-rack-controller/2.3_secure_headers.patch @@ -1,2 +1,12 @@ -170d169 -< self.setHeader(b'server', version) +diff --git a/src/twisted/web/server.py b/src/twisted/web/server.py +index 3a3f9f89b..1eb273816 100644 +--- a/src/twisted/web/server.py ++++ b/src/twisted/web/server.py +@@ -174,7 +174,6 @@ class Request(Copyable, http.Request, components.Componentized): + self.site = self.channel.site + + # set various default headers +- self.setHeader(b'server', version) + self.setHeader(b'date', http.datetimeToString()) + + # Resource Identification diff --git a/images/maas-rack-controller/Dockerfile b/images/maas-rack-controller/Dockerfile index 2870066..5e1a834 100644 --- a/images/maas-rack-controller/Dockerfile +++ b/images/maas-rack-controller/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM=ubuntu:16.04 +ARG FROM=ubuntu:18.04 FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' @@ -18,64 +18,54 @@ ARG no_proxy ENV DEBIAN_FRONTEND noninteractive ENV container docker -# everything else below is to setup maas into the systemd initialized -# container based on ubuntu 16.04 -RUN apt-get -qq update && \ - apt-get -y install \ - sudo \ - software-properties-common \ - libvirt-bin \ - systemd \ - patch \ - jq -# Don't start any optional services except for the few we need. +ENV MAAS_VERSION 2.8.6-8602-g.07cdffcaa-0ubuntu1~18.04.1 -RUN find /etc/systemd/system \ +RUN apt-get -qq update \ + && apt-get install -y \ + avahi-daemon \ + isc-dhcp-server \ + jq \ + libvirt-bin \ + patch \ + software-properties-common \ + sudo \ + systemd \ +# Don't start any optional services except for the few we need. +# (specifically, don't start avahi-daemon, isc-dhcp-server, or libvirtd) + && find /etc/systemd/system \ /lib/systemd/system \ -path '*.wants/*' \ -not -name '*journald*' \ -not -name '*systemd-tmpfiles*' \ -not -name '*systemd-user-sessions*' \ - -exec rm \{} \; -RUN systemctl set-default multi-user.target + -exec rm \{} \; \ + && systemctl set-default multi-user.target \ +# Install maas from the ppa + && add-apt-repository -yu ppa:maas/2.8 \ + && apt-get install -y \ + maas-rack-controller=$MAAS_VERSION \ + && rm -rf /var/lib/apt/lists/* -# install syslog and enable it -RUN apt-get install -y rsyslog -RUN systemctl enable rsyslog.service - -ENV MAAS_VERSION 2.3.5-6511-gf466fdb-0ubuntu1 - -# install maas -RUN rsyslogd; apt-get install -y maas-cli=$MAAS_VERSION maas-rack-controller=$MAAS_VERSION - -RUN mv /usr/sbin/tcpdump /usr/bin/tcpdump -RUN ln -s /usr/bin/tcpdump /usr/sbin/tcpdump +# Preserve the directory structure, permissions, and contents of /var/lib/maas +RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas # register ourselves with the region controller COPY scripts/register-rack-controller.service /lib/systemd/system/register-rack-controller.service RUN systemctl enable register-rack-controller.service # Patch so that Calico interfaces are ignored -# dc6350: this appears to be fixed in maas master as of 10/4/2018, but that change is not in 2.3.5 COPY 2.3_nic_filter.patch /tmp/2.3_nic_filter.patch -# sh8121att: patch so that interfaces with MAC 00:00:00:00:00:00 omit the MAC address -COPY 2.3_mac_address.patch /tmp/2.3_mac_address.patch -# sh8121att: patch so query for RPC info contains proper Host header -copy 2.3_hostheader.patch /tmp/2.3_hostheader.patch COPY 2.3_secure_headers.patch /tmp/2.3_secure_headers.patch # Patch so maas knows that "BMC error" is retriable COPY 2.3_ipmi_error.patch /tmp/2.3_ipmi_error.patch RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch network.py < /tmp/2.3_nic_filter.patch -RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch -RUN cd /usr/lib/python3/dist-packages/provisioningserver/rpc && patch clusterservice.py < /tmp/2.3_hostheader.patch RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.3_secure_headers.patch RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch ipmi.py < /tmp/2.3_ipmi_error.patch # echo journalctl logs to the container's stdout COPY scripts/journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service -RUN mkdir -p /etc/systemd/system/basic.target.wants ;\ - ln -s /etc/systemd/system/journalctl-to-tty.service /etc/systemd/system/basic.target.wants/journalctl-to-tty.service +RUN systemctl enable journalctl-to-tty.service # quiet sudo for the maas user RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log diff --git a/images/maas-region-controller/2.3_bios_grub_partition.patch b/images/maas-region-controller/2.3_bios_grub_partition.patch index 854b0fa..1ef0349 100644 --- a/images/maas-region-controller/2.3_bios_grub_partition.patch +++ b/images/maas-region-controller/2.3_bios_grub_partition.patch @@ -1,23 +1,17 @@ ---- partition.py 2018-10-18 09:04:40.300721829 -0700 -+++ partition2.py 2018-10-18 09:08:27.952565248 -0700 -@@ -179,6 +179,8 @@ - - def get_partition_number(self): - """Return the partition number in the table.""" -+ # Circular imports. -+ from maasserver.models.partitiontable import GPT_REQUIRED_SIZE - # Sort manually instead of with `order_by`, this will prevent django - # from making a query if the partitions are already cached. - partitions_in_table = self.partition_table.partitions.all() -@@ -196,7 +198,10 @@ - block_device = self.partition_table.block_device - if (arch == "ppc64el" and block_device.id == boot_disk.id): +diff --git a/src/maasserver/models/partition.py b/src/maasserver/models/partition.py +index 814660ac2..1ba4142bf 100644 +--- a/src/maasserver/models/partition.py ++++ b/src/maasserver/models/partition.py +@@ -238,7 +238,11 @@ class Partition(CleanSave, TimestampedModel): + return idx + 1 + elif arch == "ppc64el" and block_device.id == boot_disk.id: return idx + 2 - elif arch == "amd64" and bios_boot_method != "uefi": -+ elif (arch == "amd64" and -+ self.partition_table.block_device.id == boot_disk.id and -+ bios_boot_method != "uefi" and -+ boot_disk.size >= GPT_REQUIRED_SIZE): - return idx + 2 - else: - return idx + 1 ++ elif ( ++ arch == "amd64" ++ and bios_boot_method != "uefi" ++ and block_device.id == boot_disk.id ++ ): + if block_device.type == "physical": + # Delay the `type` check because it can cause a query. Only + # physical block devices get the bios_grub partition. diff --git a/images/maas-region-controller/2.3_bios_grub_preseed.patch b/images/maas-region-controller/2.3_bios_grub_preseed.patch deleted file mode 100644 index bbc9db9..0000000 --- a/images/maas-region-controller/2.3_bios_grub_preseed.patch +++ /dev/null @@ -1,22 +0,0 @@ ---- preseed_storage.py 2018-10-18 09:09:24.372519602 -0700 -+++ preseed_storage2.py 2018-10-18 09:10:30.112463704 -0700 -@@ -22,6 +22,7 @@ - from maasserver.models.partition import Partition - from maasserver.models.partitiontable import ( - BIOS_GRUB_PARTITION_SIZE, -+ GPT_REQUIRED_SIZE, - INITIAL_PARTITION_OFFSET, - PARTITION_TABLE_EXTRA_SPACE, - PREP_PARTITION_SIZE, -@@ -127,7 +128,10 @@ - """Return True if block device requires the bios_grub partition.""" - arch, _ = self.node.split_arch() - bios_boot_method = self.node.get_bios_boot_method() -- return arch == "amd64" and bios_boot_method != "uefi" -+ return ( -+ arch == "amd64" and -+ bios_boot_method != "uefi" and -+ block_device.size >= GPT_REQUIRED_SIZE) - - def _add_partition_operations(self): - """Add all the partition operations. diff --git a/images/maas-region-controller/2.3_configure_ipmi_user.patch b/images/maas-region-controller/2.3_configure_ipmi_user.patch index a82ee50..f9e909f 100644 --- a/images/maas-region-controller/2.3_configure_ipmi_user.patch +++ b/images/maas-region-controller/2.3_configure_ipmi_user.patch @@ -1,8 +1,8 @@ diff --git a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py -index e2c3ce5..7370963 100644 +index 13188ecb8..7b3dad4d4 100755 --- a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py +++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py -@@ -229,17 +229,40 @@ def make_ipmi_user_settings(username, password): +@@ -235,8 +235,30 @@ def make_ipmi_user_settings(username, password): return user_settings @@ -30,9 +30,10 @@ index e2c3ce5..7370963 100644 def configure_ipmi_user(username): """Create or configure an IPMI user for remote use.""" + exceptions_caught = [] - for password in [generate_random_password(), - generate_random_password(with_special_chars=True)]: - user_settings = make_ipmi_user_settings(username, password) + for password in [ + generate_random_password(), + generate_random_password(with_special_chars=True), +@@ -245,9 +267,11 @@ def configure_ipmi_user(username): try: apply_ipmi_user_settings(user_settings) return password @@ -42,12 +43,13 @@ index e2c3ce5..7370963 100644 + except subprocess.CalledProcessError as e: + exceptions_caught.append(e) + raise IPMIError( -+ "Unable to set BMC password:\n{}".format(exceptions_caught)) ++ "Unable to set BMC password:\n{}".format(exceptions_caught) ++ ) def set_ipmi_lan_channel_settings(): -@@ -357,7 +380,7 @@ def main(): - IPMI_MAAS_USER = "maas" +@@ -389,7 +413,7 @@ def main(): + IPMI_MAAS_USER = args.maas_ipmi_user IPMI_MAAS_PASSWORD = None - IPMI_MAAS_PASSWORD = configure_ipmi_user(IPMI_MAAS_USER) diff --git a/images/maas-region-controller/2.3_kernel_package.patch b/images/maas-region-controller/2.3_kernel_package.patch index 423f4f4..a237e8e 100644 --- a/images/maas-region-controller/2.3_kernel_package.patch +++ b/images/maas-region-controller/2.3_kernel_package.patch @@ -1,6 +1,8 @@ ---- preseed.py 2018-10-15 12:52:41.611027184 -0700 -+++ preseed2.py 2018-10-19 07:26:10.985282758 -0700 -@@ -317,7 +317,25 @@ +diff --git a/src/maasserver/preseed.py b/src/maasserver/preseed.py +index 36c656775..5da8f0312 100644 +--- a/src/maasserver/preseed.py ++++ b/src/maasserver/preseed.py +@@ -245,7 +245,27 @@ def compose_curtin_kernel_preseed(node): The BootResourceFile table contains a mapping between hwe kernels and Ubuntu package names. If this mapping is missing we fall back to letting Curtin figure out which kernel should be installed""" @@ -19,10 +21,12 @@ + # parse the string and find our package param value + # e.g. kernel_package=linux-image-4.15.0-34-generic + kparams = kernel_opts.split() -+ kdict = dict(kparam.split('=',1) for kparam in kparams if '=' in kparam) -+ if 'kernel_package' in kdict: -+ kpackage = kdict['kernel_package'] ++ kdict = dict( ++ kparam.split("=", 1) for kparam in kparams if "=" in kparam ++ ) ++ if "kernel_package" in kdict: ++ kpackage = kdict["kernel_package"] + if kpackage: - kernel_config = { - 'kernel': { + kernel_config = {"kernel": {"package": kpackage, "mapping": {}}} + return [yaml.safe_dump(kernel_config)] diff --git a/images/maas-region-controller/2.3_maas_enlist.patch b/images/maas-region-controller/2.3_maas_enlist.patch deleted file mode 100644 index 3f78cc2..0000000 --- a/images/maas-region-controller/2.3_maas_enlist.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/metadataserver/user_data/templates/snippets/maas_enlist.sh b/src/metadataserver/user_data/templates/snippets/maas_enlist.sh -index cb9515c..394b0bd 100644 ---- a/src/metadataserver/user_data/templates/snippets/maas_enlist.sh -+++ b/src/metadataserver/user_data/templates/snippets/maas_enlist.sh -@@ -236,7 +236,7 @@ fi - if echo "$serverurl" | egrep -q '(^[a-z]+://|^)[a-zA-Z0-9\.\-]+($|/$)'; then - api_url="MAAS/api/2.0/machines/" - else -- api_url=`echo $serverurl | sed 's#^\(\|[a-z]\+://\)\([a-zA-Z0-9\.]\+\|\(\[[0-9a-fA-F:]\+\]\)\)\(\|\:[0-9]\+\)/##'` -+ api_url=`echo $serverurl | sed 's#^\(\|[a-z]\+://\)\([a-zA-Z0-9\.\-]\+\|\(\[[0-9a-fA-F:]\+\]\)\)\(\|\:[0-9]\+\)/##'` - fi - - if [ -z "$hostname" ] && [ "$quite" != "true" ]; then diff --git a/images/maas-region-controller/2.3_mac_address.patch b/images/maas-region-controller/2.3_mac_address.patch deleted file mode 100644 index c709316..0000000 --- a/images/maas-region-controller/2.3_mac_address.patch +++ /dev/null @@ -1,6 +0,0 @@ -394a395,399 -> # Exclude interfaces that have duplicate MACs -> # such as OVS gretap and erspan interfaces -> ifaces = { k: v -> for k, v in ifaces.items() -> if v.get('mac', '') != '00:00:00:00:00:00'} diff --git a/images/maas-region-controller/2.3_partitiontable_does_not_exist.patch b/images/maas-region-controller/2.3_partitiontable_does_not_exist.patch index 3acb0ae..b0f98d3 100644 --- a/images/maas-region-controller/2.3_partitiontable_does_not_exist.patch +++ b/images/maas-region-controller/2.3_partitiontable_does_not_exist.patch @@ -1,11 +1,11 @@ diff --git a/src/maasserver/api/partitions.py b/src/maasserver/api/partitions.py -index fb75600a8..f03fc5685 100644 +index 5a413fb3b..6a5ad8b47 100644 --- a/src/maasserver/api/partitions.py +++ b/src/maasserver/api/partitions.py -@@ -89,7 +89,7 @@ class PartitionsHandler(OperationsHandler): - """ +@@ -99,7 +99,7 @@ class PartitionsHandler(OperationsHandler): device = BlockDevice.objects.get_block_device_or_404( - system_id, device_id, request.user, NODE_PERMISSION.VIEW) + system_id, device_id, request.user, NodePermission.view + ) - partition_table = device.partitiontable_set.get() + partition_table = device.get_partitiontable() if partition_table is None: diff --git a/images/maas-region-controller/2.3_region_secret_rotate.patch b/images/maas-region-controller/2.3_region_secret_rotate.patch index d22c5f8..92d795c 100644 --- a/images/maas-region-controller/2.3_region_secret_rotate.patch +++ b/images/maas-region-controller/2.3_region_secret_rotate.patch @@ -1,20 +1,21 @@ diff --git a/src/maasserver/security.py b/src/maasserver/security.py -index e931b89..44fad1a 100644 +index f92529265..542970009 100644 --- a/src/maasserver/security.py +++ b/src/maasserver/security.py -@@ -105,10 +105,11 @@ def get_shared_secret_txn(): +@@ -97,11 +97,11 @@ def get_shared_secret_txn(): elif secret_in_db == secret_on_fs: secret = secret_in_db # or secret_on_fs. else: - raise AssertionError( - "The secret stored in the database does not match the secret " -- "stored on the filesystem at %s. Please investigate." % -- get_shared_secret_filesystem_path()) +- "stored on the filesystem at %s. Please investigate." +- % get_shared_secret_filesystem_path() +- ) + # (nk613n): When we rotate secrets we only update the filesystem + # so if the secrets don't match we will default to the FS + # secret and set it in the database (set_config function) + secret = secret_on_fs + Config.objects.set_config("rpc_shared_secret", to_hex(secret)) - + return secret - + diff --git a/images/maas-region-controller/2.3_route.patch b/images/maas-region-controller/2.3_route.patch index f5e9765..e7a3b34 100644 --- a/images/maas-region-controller/2.3_route.patch +++ b/images/maas-region-controller/2.3_route.patch @@ -1,13 +1,17 @@ diff --git a/src/maasserver/preseed_network.py b/src/maasserver/preseed_network.py -index bb56765..1185ff1 100644 +index 99a3ce309..2a9e72d88 100644 --- a/src/maasserver/preseed_network.py +++ b/src/maasserver/preseed_network.py -@@ -203,7 +203,7 @@ class InterfaceConfiguration: - return { - route - for route in self.routes -- if route.source == source +@@ -308,7 +308,11 @@ class InterfaceConfiguration: + + def _get_matching_routes(self, source): + """Return all route objects matching `source`.""" +- return {route for route in self.routes if route.source == source} ++ return { ++ route ++ for route in self.routes + if str(route.source.cidr) == str(source.cidr) - } - ++ } + def _generate_addresses(self, version=1): + """Generate the various addresses needed for this interface.""" diff --git a/images/maas-region-controller/2.3_secure_headers.patch b/images/maas-region-controller/2.3_secure_headers.patch index 6c006eb..c7a520e 100644 --- a/images/maas-region-controller/2.3_secure_headers.patch +++ b/images/maas-region-controller/2.3_secure_headers.patch @@ -1,2 +1,12 @@ -170d169 -< self.setHeader(b'server', version) +diff --git a/src/twisted/web/server.py b/src/twisted/web/server.py +index 3a3f9f89b..1eb273816 100644 +--- a/src/twisted/web/server.py ++++ b/src/twisted/web/server.py +@@ -174,7 +174,6 @@ class Request(Copyable, http.Request, components.Componentized): + self.site = self.channel.site + + # set various default headers +- self.setHeader(b'server', version) + self.setHeader(b'date', http.datetimeToString()) + + # Resource Identification diff --git a/images/maas-region-controller/2.8_maas_ipmi_autodetect_tool.patch b/images/maas-region-controller/2.8_maas_ipmi_autodetect_tool.patch new file mode 100644 index 0000000..2175681 --- /dev/null +++ b/images/maas-region-controller/2.8_maas_ipmi_autodetect_tool.patch @@ -0,0 +1,17 @@ +diff --git a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect_tool.py b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect_tool.py +index f8ca88467..530bc7d15 100755 +--- a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect_tool.py ++++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect_tool.py +@@ -33,7 +33,11 @@ def detect_ipmi(): + + + def is_host_moonshot(): +- output = subprocess.check_output(["ipmitool", "raw", "06", "01"]) ++ (status, output) = subprocess.getstatusoutput( ++ ["ipmitool", "raw", "06", "01"] ++ ) ++ if status != 0: ++ return False + # 14 is the code that identifies a machine as a moonshot + if output.split()[0] == "14": + return True diff --git a/images/maas-region-controller/Dockerfile b/images/maas-region-controller/Dockerfile index 29be3a9..600d544 100644 --- a/images/maas-region-controller/Dockerfile +++ b/images/maas-region-controller/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM=ubuntu:16.04 +ARG FROM=ubuntu:18.04 FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' @@ -18,92 +18,65 @@ ARG no_proxy ENV DEBIAN_FRONTEND noninteractive ENV container docker +ENV MAAS_VERSION 2.8.6-8602-g.07cdffcaa-0ubuntu1~18.04.1 + +RUN apt-get -qq update \ + && apt-get install -y \ + avahi-daemon \ + jq \ + patch \ + software-properties-common \ + sudo \ + systemd \ # Don't start any optional services except for the few we need. -RUN find /etc/systemd/system \ +# (specifically, don't start avahi-daemon) + && find /etc/systemd/system \ /lib/systemd/system \ -path '*.wants/*' \ -not -name '*journald*' \ -not -name '*systemd-tmpfiles*' \ -not -name '*systemd-user-sessions*' \ - -exec rm \{} \; -RUN systemctl set-default multi-user.target + -exec rm \{} \; \ + && systemctl set-default multi-user.target \ +# Install maas from the ppa + && add-apt-repository -yu ppa:maas/2.8 \ + && apt-get install -y \ + maas-region-api=$MAAS_VERSION \ + # tcpdump is required by /usr/lib/maas/beacon-monitor + tcpdump \ + && rm -rf /var/lib/apt/lists/* -# everything else below is to setup maas into the systemd initialized -# container based on ubuntu 16.04 -RUN apt-get -qq update && \ - apt-get -y install sudo \ - software-properties-common \ - jq - -# TODO(alanmeadows) -# we need systemd 231 per https://github.com/systemd/systemd/commit/a1350640ba605cf5876b25abfee886488a33e50b -#RUN add-apt-repository ppa:pitti/systemd -y && add-apt-repository ppa:maas/stable -y && apt-get update -RUN apt-get install -y systemd - -# install syslog and enable it -RUN apt-get install -y rsyslog -RUN systemctl enable rsyslog.service - -ENV MAAS_VERSION 2.3.5-6511-gf466fdb-0ubuntu1 - -# install maas -RUN rsyslogd; apt-get install -y maas-cli=$MAAS_VERSION \ - maas-dns=$MAAS_VERSION \ - maas-region-api=$MAAS_VERSION \ - avahi-utils \ - dbconfig-pgsql=2.0.4ubuntu1 \ - iputils-ping \ - postgresql \ - tcpdump \ - python3-pip - - -RUN apt-get download maas-region-controller=$MAAS_VERSION && \ -# remove postinstall script in order to avoid db_sync - dpkg-deb --extract maas-region-controller*.deb maas-region-controller && \ - dpkg-deb --control maas-region-controller*.deb maas-region-controller/DEBIAN && \ - rm maas-region-controller/DEBIAN/postinst && \ - dpkg-deb --build maas-region-controller && \ - dpkg -i maas-region-controller.deb && \ - pg_dropcluster --stop 9.5 main - -# we don't want/need avahi-daemon running in the container -RUN rm -f /etc/init.d/avahi-daemon +# Preserve the directory structure, permissions, and contents of /var/lib/maas +RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas # 2.3 workarounds COPY 2.3_route.patch /tmp/2.3_route.patch COPY 2.3_kernel_package.patch /tmp/2.3_kernel_package.patch COPY 2.3_bios_grub_partition.patch /tmp/2.3_bios_grub_partition.patch -COPY 2.3_bios_grub_preseed.patch /tmp/2.3_bios_grub_preseed.patch -# sh8121att: patch so that maas-enlist works with domains that contain '-' -COPY 2.3_maas_enlist.patch /tmp/2.3_maas_enlist.patch -# sh8121att: patch so that interfaces with MAC 00:00:00:00:00:00 omit the MAC address -COPY 2.3_mac_address.patch /tmp/2.3_mac_address.patch # sh8121att: allow all requests via the proxy to allow it to work # behind ingress COPY 2.3_proxy_acl.patch /tmp/2.3_proxy_acl.patch # Patch to add retrying to MaaS BMC user setup, and improve exception handling -copy 2.3_configure_ipmi_user.patch /tmp/2.3_configure_ipmi_user.patch +COPY 2.3_configure_ipmi_user.patch /tmp/2.3_configure_ipmi_user.patch COPY 2.3_secure_headers.patch /tmp/2.3_secure_headers.patch COPY 2.3_region_secret_rotate.patch /tmp/2.3_region_secret_rotate.patch COPY 2.3_partitiontable_does_not_exist.patch /tmp/2.3_partitiontable_does_not_exist.patch +# Avoid enlistment failures due to exceptions during moonshot detect attempts +COPY 2.8_maas_ipmi_autodetect_tool.patch /tmp/2.8_maas_ipmi_autodetect_tool.patch RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.3_route.patch RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.3_kernel_package.patch RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.3_bios_grub_partition.patch -RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_storage.py < /tmp/2.3_bios_grub_preseed.patch RUN cd /usr/lib/python3/dist-packages/maasserver && patch security.py < /tmp/2.3_region_secret_rotate.patch -RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_enlist.sh < /tmp/2.3_maas_enlist.patch RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_ipmi_autodetect.py < /tmp/2.3_configure_ipmi_user.patch -RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.3_proxy_acl.patch RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.3_secure_headers.patch RUN cd /usr/lib/python3/dist-packages/maasserver/api && patch partitions.py < /tmp/2.3_partitiontable_does_not_exist.patch +RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets/ && patch maas_ipmi_autodetect_tool.py < /tmp/2.8_maas_ipmi_autodetect_tool.patch - +# echo journalctl logs to the container's stdout COPY journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service -RUN mkdir -p /etc/systemd/system/basic.target.wants ;\ - ln -s /etc/systemd/system/journalctl-to-tty.service /etc/systemd/system/basic.target.wants/journalctl-to-tty.service +RUN systemctl enable journalctl-to-tty.service # quiet sudo for the maas user RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log diff --git a/images/sstream-cache/Dockerfile b/images/sstream-cache/Dockerfile index 9e5ca1f..d5b3b24 100644 --- a/images/sstream-cache/Dockerfile +++ b/images/sstream-cache/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM=ubuntu:16.04 +ARG FROM=ubuntu:18.04 FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' @@ -15,8 +15,8 @@ ARG http_proxy ARG https_proxy ARG no_proxy -ARG SSTREAM_IMAGE=https://images.maas.io/ephemeral-v3/daily/ -ARG SSTREAM_RELEASE=xenial +ARG SSTREAM_IMAGE=https://images.maas.io/ephemeral-v3/stable/ +ARG SSTREAM_RELEASE=bionic RUN apt-get -qq update && \ apt install -y simplestreams \