From 48df9fd6f5bafcc80a10d0767c240a73362c3a43 Mon Sep 17 00:00:00 2001
From: "Carter, Matt (mc981n)" <mc981n@att.com>
Date: Mon, 16 Sep 2019 14:20:20 -0500
Subject: [PATCH] Add retries to MaaS BMC user configuration

It has been observed that MaaS will fail to enlist/commission/deploy
nodes if it fails to set up its own user in the BMC during cloud
init. This patch set adds a git patch file to update the MaaS source
code in order to retry setting up the MaaS BMC user if it fails.

This patch set also adds to the exception message sent when MaaS
fails to set up a BMC user.

Change-Id: I475988875acffac620302fae3eed8d236a5a46f7
---
 .../2.3_configure_ipmi_user.patch             | 57 +++++++++++++++++++
 images/maas-region-controller/Dockerfile      |  4 ++
 2 files changed, 61 insertions(+)
 create mode 100644 images/maas-region-controller/2.3_configure_ipmi_user.patch

diff --git a/images/maas-region-controller/2.3_configure_ipmi_user.patch b/images/maas-region-controller/2.3_configure_ipmi_user.patch
new file mode 100644
index 0000000..a82ee50
--- /dev/null
+++ b/images/maas-region-controller/2.3_configure_ipmi_user.patch
@@ -0,0 +1,57 @@
+diff --git a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
+index e2c3ce5..7370963 100644
+--- a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
++++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
+@@ -229,17 +229,40 @@ def make_ipmi_user_settings(username, password):
+     return user_settings
+ 
+ 
++def configure_ipmi_user_with_backoff(username):
++    """Create/configure an IPMI user, but with several tries"""
++    attempt = 1
++    max_attempts = 5
++    backoff_amount = 30
++    while attempt <= max_attempts:
++        password = None
++        try:
++            password = configure_ipmi_user(username)
++        except:
++            if (attempt + 1) > max_attempts:
++                # This is our last attempt, don't catch anything:
++                raise
++
++        if password is None:
++            time.sleep(attempt * backoff_amount)
++        else:
++            return password
++        attempt += 1
++
++
+ def configure_ipmi_user(username):
+     """Create or configure an IPMI user for remote use."""
++    exceptions_caught = []
+     for password in [generate_random_password(),
+                      generate_random_password(with_special_chars=True)]:
+         user_settings = make_ipmi_user_settings(username, password)
+         try:
+             apply_ipmi_user_settings(user_settings)
+             return password
+-        except subprocess.CalledProcessError:
+-            pass
+-    raise IPMIError("Unable to set BMC password.")
++        except subprocess.CalledProcessError as e:
++            exceptions_caught.append(e)
++    raise IPMIError(
++        "Unable to set BMC password:\n{}".format(exceptions_caught))
+ 
+ 
+ def set_ipmi_lan_channel_settings():
+@@ -357,7 +380,7 @@ def main():
+     IPMI_MAAS_USER = "maas"
+     IPMI_MAAS_PASSWORD = None
+ 
+-    IPMI_MAAS_PASSWORD = configure_ipmi_user(IPMI_MAAS_USER)
++    IPMI_MAAS_PASSWORD = configure_ipmi_user_with_backoff(IPMI_MAAS_USER)
+ 
+     # Attempt to enable IPMI Over Lan. If it is disabled, MAAS won't
+     # be able to remotely communicate to the BMC.
diff --git a/images/maas-region-controller/Dockerfile b/images/maas-region-controller/Dockerfile
index 2552422..f413366 100644
--- a/images/maas-region-controller/Dockerfile
+++ b/images/maas-region-controller/Dockerfile
@@ -72,11 +72,15 @@ COPY 2.3_mac_address.patch /tmp/2.3_mac_address.patch
 # sh8121att: allow all requests via the proxy to allow it to work
 # behind ingress
 COPY 2.3_proxy_acl.patch /tmp/2.3_proxy_acl.patch
+# Patch to add retrying to MaaS BMC user setup, and improve exception handling
+copy 2.3_configure_ipmi_user.patch /tmp/2.3_configure_ipmi_user.patch
+
 RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.3_route.patch
 RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.3_kernel_package.patch
 RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.3_bios_grub_partition.patch
 RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_storage.py < /tmp/2.3_bios_grub_preseed.patch
 RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_enlist.sh < /tmp/2.3_maas_enlist.patch
+RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_ipmi_autodetect.py < /tmp/2.3_configure_ipmi_user.patch
 RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch
 RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch
 RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.3_proxy_acl.patch