MAAS region & rack controller upgrade v3.0.0

Signed-off-by: Ruslan Aliev <raliev@mirantis.com>
Change-Id: I87a86c920e49e53447c87bcff3c0fae08ebf267f

images/maas-rack-controller-focal/2.8_ipmi_error.patch

@@ -0,0 +1,27 @@
+diff --git a/src/provisioningserver/drivers/power/ipmi.py b/src/provisioningserver/drivers/power/ipmi.py
+index e99b807ce..8f56dc77a 100644
+--- a/src/provisioningserver/drivers/power/ipmi.py
++++ b/src/provisioningserver/drivers/power/ipmi.py
+@@ -154,6 +154,13 @@ IPMI_ERRORS = {
+         ),
+         "exception": PowerConnError,
+     },
++    "BMC error": {
++        "message": (
++            "Device not responding correctly while performing power action."
++            "  MAAS performed several retries.  Please wait and try again."
++        ),
++        "exception": PowerConnError,
++    },
+     "could not find inband device": {
+         "message": (
+             "An inband device could not be found."
+@@ -231,7 +238,7 @@ class IPMIPowerDriver(PowerDriver):
+         ),
+     ]
+     ip_extractor = make_ip_extractor("power_address")
+-    wait_time = (4, 8, 16, 32)
++    wait_time = (4, 4, 8, 8, 16, 16, 32, 32)
+ 
+     def detect_missing_packages(self):
+         if not shell.has_command_available("ipmipower"):

images/maas-rack-controller-focal/2.8_nic_filter.patch

@@ -0,0 +1,13 @@
+diff --git a/src/provisioningserver/utils/network.py b/src/provisioningserver/utils/network.py
+index 55f8449f4..ebde045ee 100644
+--- a/src/provisioningserver/utils/network.py
++++ b/src/provisioningserver/utils/network.py
+@@ -1153,6 +1153,8 @@ def get_all_interfaces_definition(
+         # interfaces for guests. By themselves, they're not useful for MAAS to
+         # manage.
+         "ethernet.tunnel",
++        # Always exclude non-specific ethernet interfaces.
++        "ethernet",
+     ]
+     if not running_in_container():
+         # When not running in a container, we should be able to identify

images/maas-rack-controller-focal/2.8_redfish_retries.patch

@@ -0,0 +1,12 @@
+diff --git a/src/provisioningserver/drivers/power/redfish.py b/src/provisioningserver/drivers/power/redfish.py
+index 27f63545a..9c39d577e 100644
+--- a/src/provisioningserver/drivers/power/redfish.py
++++ b/src/provisioningserver/drivers/power/redfish.py
+@@ -149,6 +149,7 @@ class RedfishPowerDriver(RedfishPowerDriverBase):
+         make_setting_field("node_id", "Node ID", scope=SETTING_SCOPE.NODE),
+     ]
+     ip_extractor = make_ip_extractor("power_address")
++    wait_time = (4, 8, 16, 32)
+ 
+     def detect_missing_packages(self):
+         # no required packages

images/maas-rack-controller-focal/Dockerfile

@@ -0,0 +1,81 @@
+ARG FROM=ubuntu:20.04
+FROM ${FROM}
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
+LABEL org.opencontainers.image.url='https://airshipit.org'
+LABEL org.opencontainers.image.documentation='https://github.com/openstack/airship-maas'
+LABEL org.opencontainers.image.source='https://git.openstack.org/openstack/airship-maas'
+LABEL org.opencontainers.image.vendor='The Airship Authors'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+ARG http_proxy
+ARG https_proxy
+ARG no_proxy
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV container docker
+
+ENV MAAS_VERSION 1:3.0.0-10029-g.986ea3e45-0ubuntu1~20.04.1
+
+RUN apt-get -qq update \
+ && apt-get install -y \
+        avahi-daemon \
+        isc-dhcp-server \
+        jq \
+        libvirt-bin \
+        patch \
+        software-properties-common \
+        sudo \
+        systemd \
+        ca-certificates \
+# Don't start any optional services except for the few we need.
+# (specifically, don't start avahi-daemon, isc-dhcp-server, or libvirtd)
+ && find /etc/systemd/system \
+         /lib/systemd/system \
+         -path '*.wants/*' \
+         -not -name '*journald*' \
+         -not -name '*systemd-tmpfiles*' \
+         -not -name '*systemd-user-sessions*' \
+         -exec rm \{} \; \
+ && systemctl set-default multi-user.target \
+# Install maas from the ppa
+ && add-apt-repository -yu ppa:maas/3.0 \
+ && apt-get install -y \
+        maas-rack-controller=$MAAS_VERSION \
+ && rm -rf /var/lib/apt/lists/*
+
+# Preserve the directory structure, permissions, and contents of /var/lib/maas
+RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas
+
+# register ourselves with the region controller
+COPY scripts/register-rack-controller.service /lib/systemd/system/register-rack-controller.service
+RUN systemctl enable register-rack-controller.service
+
+# Patch so that Calico interfaces are ignored
+COPY 2.8_nic_filter.patch /tmp/2.8_nic_filter.patch
+COPY 2.8_secure_headers.patch /tmp/2.8_secure_headers.patch
+# Patch so maas knows that "BMC error" is retriable
+COPY 2.8_ipmi_error.patch /tmp/2.8_ipmi_error.patch
+# Patch to space redfish request retries apart a bit, to avoid overwhelming the BMC
+COPY 2.8_redfish_retries.patch /tmp/2.8_redfish_retries.patch
+
+#RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch network.py < /tmp/2.8_nic_filter.patch
+#RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.8_secure_headers.patch
+#RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch ipmi.py < /tmp/2.8_ipmi_error.patch
+#RUN cd /usr/lib/python3/dist-packages/provisioningserver/drivers/power && patch redfish.py < /tmp/2.8_redfish_retries.patch
+
+# echo journalctl logs to the container's stdout
+COPY scripts/journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service
+RUN systemctl enable journalctl-to-tty.service
+
+# quiet sudo for the maas user
+RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log
+
+# avoid triggering bind9 high cpu utilization bug
+RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf
+
+# initalize systemd
+CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]

images/maas-rack-controller-focal/README.md

@@ -0,0 +1 @@
+[![Docker Repository on Quay](https://quay.io/repository/airshipit/maas-rack/status "Docker Repository on Quay")](https://quay.io/repository/airshipit/maas-rack) Ubuntu MaaS Rack Controller

images/maas-rack-controller-focal/scripts/register-rack-controller.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Register with MaaS Region Controller
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+PassEnvironment=MAAS_ENDPOINT MAAS_REGION_SECRET MAAS_API_KEY HOST_MOUNT_PATH
+ExecStart=/usr/local/bin/register-rack-controller.sh
+
+[Install]
+WantedBy=multi-user.target

images/maas-region-controller-bionic/2.8_secure_headers.patch

@@ -0,0 +1,12 @@
+diff --git a/src/twisted/web/server.py b/src/twisted/web/server.py
+index 3a3f9f89b..1eb273816 100644
+--- a/src/twisted/web/server.py
++++ b/src/twisted/web/server.py
+@@ -174,7 +174,6 @@ class Request(Copyable, http.Request, components.Componentized):
+         self.site = self.channel.site
+ 
+         # set various default headers
+-        self.setHeader(b'server', version)
+         self.setHeader(b'date', http.datetimeToString())
+ 
+         # Resource Identification

images/maas-region-controller-bionic/journalctl-to-tty.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Journald console log streamer
+Requires=systemd-journald.service
+After=systemd-journald.service
+
+[Service]
+Restart=always
+RestartSec=0
+ExecStart=/bin/journalctl -f
+StandardOutput=tty
+
+[Install]
+WantedBy=basic.target

images/maas-region-controller-focal/2.8_bios_grub_partition.patch

@@ -0,0 +1,17 @@
+diff --git a/src/maasserver/models/partition.py b/src/maasserver/models/partition.py
+index 814660ac2..1ba4142bf 100644
+--- a/src/maasserver/models/partition.py
++++ b/src/maasserver/models/partition.py
+@@ -238,7 +238,11 @@ class Partition(CleanSave, TimestampedModel):
+                     return idx + 1
+             elif arch == "ppc64el" and block_device.id == boot_disk.id:
+                 return idx + 2
+-            elif arch == "amd64" and bios_boot_method != "uefi":
++            elif (
++                arch == "amd64"
++                and bios_boot_method != "uefi"
++                and block_device.id == boot_disk.id
++            ):
+                 if block_device.type == "physical":
+                     # Delay the `type` check because it can cause a query. Only
+                     # physical block devices get the bios_grub partition.

images/maas-region-controller-focal/2.8_configure_ipmi_user.patch

@@ -0,0 +1,59 @@
+diff --git a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
+index 13188ecb8..7b3dad4d4 100755
+--- a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
++++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
+@@ -235,8 +235,30 @@ def make_ipmi_user_settings(username, password):
+     return user_settings
+ 
+ 
++def configure_ipmi_user_with_backoff(username):
++    """Create/configure an IPMI user, but with several tries"""
++    attempt = 1
++    max_attempts = 5
++    backoff_amount = 30
++    while attempt <= max_attempts:
++        password = None
++        try:
++            password = configure_ipmi_user(username)
++        except:
++            if (attempt + 1) > max_attempts:
++                # This is our last attempt, don't catch anything:
++                raise
++
++        if password is None:
++            time.sleep(attempt * backoff_amount)
++        else:
++            return password
++        attempt += 1
++
++
+ def configure_ipmi_user(username):
+     """Create or configure an IPMI user for remote use."""
++    exceptions_caught = []
+     for password in [
+         generate_random_password(),
+         generate_random_password(with_special_chars=True),
+@@ -245,9 +267,11 @@ def configure_ipmi_user(username):
+         try:
+             apply_ipmi_user_settings(user_settings)
+             return password
+-        except subprocess.CalledProcessError:
+-            pass
+-    raise IPMIError("Unable to set BMC password.")
++        except subprocess.CalledProcessError as e:
++            exceptions_caught.append(e)
++    raise IPMIError(
++        "Unable to set BMC password:\n{}".format(exceptions_caught)
++    )
+ 
+ 
+ def set_ipmi_lan_channel_settings():
+@@ -389,7 +413,7 @@ def main():
+     IPMI_MAAS_USER = args.maas_ipmi_user
+     IPMI_MAAS_PASSWORD = None
+ 
+-    IPMI_MAAS_PASSWORD = configure_ipmi_user(IPMI_MAAS_USER)
++    IPMI_MAAS_PASSWORD = configure_ipmi_user_with_backoff(IPMI_MAAS_USER)
+ 
+     # Attempt to enable IPMI Over Lan. If it is disabled, MAAS won't
+     # be able to remotely communicate to the BMC.

images/maas-region-controller-focal/2.8_kernel_package.patch

@@ -0,0 +1,32 @@
+diff --git a/src/maasserver/preseed.py b/src/maasserver/preseed.py
+index 36c656775..5da8f0312 100644
+--- a/src/maasserver/preseed.py
++++ b/src/maasserver/preseed.py
+@@ -245,7 +245,27 @@ def compose_curtin_kernel_preseed(node):
+     The BootResourceFile table contains a mapping between hwe kernels and
+     Ubuntu package names. If this mapping is missing we fall back to letting
+     Curtin figure out which kernel should be installed"""
++
++    # previous logic to retrieve kpackage parameter
+     kpackage = BootResource.objects.get_kpackage_for_node(node)
++
++    # determine if this node has kernel parameters applied by drydock
++    # and override kpackage if we discover the right properties
++    kernel_opt_tag = "%s_kp" % (node.hostname)
++    if kernel_opt_tag in node.tag_names():
++
++        # the tag exists, retrieve it
++        kernel_opts = node.tags.get(name=kernel_opt_tag).kernel_opts
++
++        # parse the string and find our package param value
++        # e.g. kernel_package=linux-image-4.15.0-34-generic
++        kparams = kernel_opts.split()
++        kdict = dict(
++            kparam.split("=", 1) for kparam in kparams if "=" in kparam
++        )
++        if "kernel_package" in kdict:
++            kpackage = kdict["kernel_package"]
++
+     if kpackage:
+         kernel_config = {"kernel": {"package": kpackage, "mapping": {}}}
+         return [yaml.safe_dump(kernel_config)]

images/maas-region-controller-focal/2.8_maas_ipmi_autodetect_tool.patch

@@ -0,0 +1,17 @@
+diff --git a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect_tool.py b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect_tool.py
+index f8ca88467..530bc7d15 100755
+--- a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect_tool.py
++++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect_tool.py
+@@ -33,7 +33,11 @@ def detect_ipmi():
+ 
+ 
+ def is_host_moonshot():
+-    output = subprocess.check_output(["ipmitool", "raw", "06", "01"])
++    (status, output) = subprocess.getstatusoutput(
++        ["ipmitool", "raw", "06", "01"]
++    )
++    if status != 0:
++        return False
+     # 14 is the code that identifies a machine as a moonshot
+     if output.split()[0] == "14":
+         return True

images/maas-region-controller-focal/2.8_partitiontable_does_not_exist.patch

@@ -0,0 +1,13 @@
+diff --git a/src/maasserver/api/partitions.py b/src/maasserver/api/partitions.py
+index 5a413fb3b..6a5ad8b47 100644
+--- a/src/maasserver/api/partitions.py
++++ b/src/maasserver/api/partitions.py
+@@ -99,7 +99,7 @@ class PartitionsHandler(OperationsHandler):
+         device = BlockDevice.objects.get_block_device_or_404(
+             system_id, device_id, request.user, NodePermission.view
+         )
+-        partition_table = device.partitiontable_set.get()
++        partition_table = device.get_partitiontable()
+         if partition_table is None:
+             return []
+         else:

images/maas-region-controller-focal/2.8_proxy_acl.patch

@@ -0,0 +1,10 @@
+18,24c18
+< http_access allow maas_proxy_manager localhost
+< http_access deny maas_proxy_manager
+< http_access deny !Safe_ports
+< http_access deny CONNECT !SSL_ports
+< http_access allow localnet
+< http_access allow localhost
+< http_access deny all
+---
+> http_access allow all

images/maas-region-controller-focal/2.8_region_secret_rotate.patch

@@ -0,0 +1,21 @@
+diff --git a/src/maasserver/security.py b/src/maasserver/security.py
+index f92529265..542970009 100644
+--- a/src/maasserver/security.py
++++ b/src/maasserver/security.py
+@@ -97,11 +97,11 @@ def get_shared_secret_txn():
+     elif secret_in_db == secret_on_fs:
+         secret = secret_in_db  # or secret_on_fs.
+     else:
+-        raise AssertionError(
+-            "The secret stored in the database does not match the secret "
+-            "stored on the filesystem at %s. Please investigate."
+-            % get_shared_secret_filesystem_path()
+-        )
++        # (nk613n): When we rotate secrets we only update the filesystem
++        #           so if the secrets don't match we will default to the FS
++        #           secret and set it in the database (set_config function)
++        secret = secret_on_fs
++        Config.objects.set_config("rpc_shared_secret", to_hex(secret))
+ 
+     return secret
+ 

images/maas-region-controller-focal/2.8_route.patch

@@ -0,0 +1,17 @@
+diff --git a/src/maasserver/preseed_network.py b/src/maasserver/preseed_network.py
+index 99a3ce309..2a9e72d88 100644
+--- a/src/maasserver/preseed_network.py
++++ b/src/maasserver/preseed_network.py
+@@ -308,7 +308,11 @@ class InterfaceConfiguration:
+ 
+     def _get_matching_routes(self, source):
+         """Return all route objects matching `source`."""
+-        return {route for route in self.routes if route.source == source}
++        return {
++            route
++            for route in self.routes
++            if str(route.source.cidr) == str(source.cidr)
++        }
+ 
+     def _generate_addresses(self, version=1):
+         """Generate the various addresses needed for this interface."""

images/maas-region-controller-focal/2.8_secure_headers.patch

@@ -0,0 +1,12 @@
+diff --git a/src/twisted/web/server.py b/src/twisted/web/server.py
+index 3a3f9f89b..1eb273816 100644
+--- a/src/twisted/web/server.py
++++ b/src/twisted/web/server.py
+@@ -174,7 +174,6 @@ class Request(Copyable, http.Request, components.Componentized):
+         self.site = self.channel.site
+ 
+         # set various default headers
+-        self.setHeader(b'server', version)
+         self.setHeader(b'date', http.datetimeToString())
+ 
+         # Resource Identification

images/maas-region-controller-focal/Dockerfile

@@ -0,0 +1,89 @@
+ARG FROM=ubuntu:20.04
+FROM ${FROM}
+
+LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode'
+LABEL org.opencontainers.image.url='https://airshipit.org'
+LABEL org.opencontainers.image.documentation='https://github.com/openstack/airship-maas'
+LABEL org.opencontainers.image.source='https://git.openstack.org/openstack/airship-maas'
+LABEL org.opencontainers.image.vendor='The Airship Authors'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+ARG http_proxy
+ARG https_proxy
+ARG no_proxy
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV container docker
+
+ENV MAAS_VERSION 1:3.0.0-10029-g.986ea3e45-0ubuntu1~20.04.1
+
+RUN apt-get -qq update \
+ && apt-get install -y \
+        avahi-daemon \
+        jq \
+        patch \
+        software-properties-common \
+        sudo \
+        systemd \
+        ca-certificates \
+# Don't start any optional services except for the few we need.
+# (specifically, don't start avahi-daemon)
+ && find /etc/systemd/system \
+         /lib/systemd/system \
+         -path '*.wants/*' \
+         -not -name '*journald*' \
+         -not -name '*systemd-tmpfiles*' \
+         -not -name '*systemd-user-sessions*' \
+         -exec rm \{} \; \
+ && systemctl set-default multi-user.target \
+# Install maas from the ppa
+ && add-apt-repository -yu ppa:maas/3.0 \
+ && apt-get install -y \
+        maas-region-api=$MAAS_VERSION \
+        # tcpdump is required by /usr/lib/maas/beacon-monitor
+        tcpdump \
+ && rm -rf /var/lib/apt/lists/*
+
+# Preserve the directory structure, permissions, and contents of /var/lib/maas
+RUN mkdir -p /opt/maas/ && tar -cvzf /opt/maas/var-lib-maas.tgz /var/lib/maas
+
+# MAAS workarounds
+COPY 2.8_route.patch /tmp/2.8_route.patch
+COPY 2.8_kernel_package.patch /tmp/2.8_kernel_package.patch
+COPY 2.8_bios_grub_partition.patch /tmp/2.8_bios_grub_partition.patch
+# sh8121att: allow all requests via the proxy to allow it to work
+# behind ingress
+COPY 2.8_proxy_acl.patch /tmp/2.8_proxy_acl.patch
+# Patch to add retrying to MaaS BMC user setup, and improve exception handling
+COPY 2.8_configure_ipmi_user.patch /tmp/2.8_configure_ipmi_user.patch
+COPY 2.8_secure_headers.patch /tmp/2.8_secure_headers.patch
+COPY 2.8_region_secret_rotate.patch /tmp/2.8_region_secret_rotate.patch
+COPY 2.8_partitiontable_does_not_exist.patch /tmp/2.8_partitiontable_does_not_exist.patch
+# Avoid enlistment failures due to exceptions during moonshot detect attempts
+COPY 2.8_maas_ipmi_autodetect_tool.patch /tmp/2.8_maas_ipmi_autodetect_tool.patch
+
+#RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.8_route.patch
+#RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.8_kernel_package.patch
+#RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.8_bios_grub_partition.patch
+#RUN cd /usr/lib/python3/dist-packages/maasserver && patch security.py < /tmp/2.8_region_secret_rotate.patch
+#RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_ipmi_autodetect.py < /tmp/2.8_configure_ipmi_user.patch
+#RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.8_proxy_acl.patch
+#RUN cd /usr/lib/python3/dist-packages/twisted/web && patch server.py < /tmp/2.8_secure_headers.patch
+#RUN cd /usr/lib/python3/dist-packages/maasserver/api && patch partitions.py < /tmp/2.8_partitiontable_does_not_exist.patch
+#RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets/ && patch maas_ipmi_autodetect_tool.py < /tmp/2.8_maas_ipmi_autodetect_tool.patch
+
+# echo journalctl logs to the container's stdout
+COPY journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service
+RUN systemctl enable journalctl-to-tty.service
+
+# quiet sudo for the maas user
+RUN umask 0337; echo 'Defaults:maas !pam_session, !syslog' > /etc/sudoers.d/99-maas-no-log
+
+# avoid triggering bind9 high cpu utilization bug
+RUN sed -i -e '$a\include "/etc/bind/bind.keys";' /etc/bind/named.conf
+
+# initalize systemd
+CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=console 3>&1"]

images/maas-region-controller-focal/README.md

@@ -0,0 +1 @@
+[![Docker Repository on Quay](https://quay.io/repository/airshipit/maas-rack/status "Docker Repository on Quay")](https://quay.io/repository/airshipit/maas-region) Ubuntu MaaS Region Controller

images/maas-region-controller-focal/journalctl-to-tty.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Journald console log streamer
+Requires=systemd-journald.service
+After=systemd-journald.service
+
+[Service]
+Restart=always
+RestartSec=0
+ExecStart=/bin/journalctl -f
+StandardOutput=tty
+
+[Install]
+WantedBy=basic.target