diff --git a/images/maas-region-controller/2.3_configure_ipmi_user.patch b/images/maas-region-controller/2.3_configure_ipmi_user.patch new file mode 100644 index 0000000..a82ee50 --- /dev/null +++ b/images/maas-region-controller/2.3_configure_ipmi_user.patch @@ -0,0 +1,57 @@ +diff --git a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py +index e2c3ce5..7370963 100644 +--- a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py ++++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py +@@ -229,17 +229,40 @@ def make_ipmi_user_settings(username, password): + return user_settings + + ++def configure_ipmi_user_with_backoff(username): ++ """Create/configure an IPMI user, but with several tries""" ++ attempt = 1 ++ max_attempts = 5 ++ backoff_amount = 30 ++ while attempt <= max_attempts: ++ password = None ++ try: ++ password = configure_ipmi_user(username) ++ except: ++ if (attempt + 1) > max_attempts: ++ # This is our last attempt, don't catch anything: ++ raise ++ ++ if password is None: ++ time.sleep(attempt * backoff_amount) ++ else: ++ return password ++ attempt += 1 ++ ++ + def configure_ipmi_user(username): + """Create or configure an IPMI user for remote use.""" ++ exceptions_caught = [] + for password in [generate_random_password(), + generate_random_password(with_special_chars=True)]: + user_settings = make_ipmi_user_settings(username, password) + try: + apply_ipmi_user_settings(user_settings) + return password +- except subprocess.CalledProcessError: +- pass +- raise IPMIError("Unable to set BMC password.") ++ except subprocess.CalledProcessError as e: ++ exceptions_caught.append(e) ++ raise IPMIError( ++ "Unable to set BMC password:\n{}".format(exceptions_caught)) + + + def set_ipmi_lan_channel_settings(): +@@ -357,7 +380,7 @@ def main(): + IPMI_MAAS_USER = "maas" + IPMI_MAAS_PASSWORD = None + +- IPMI_MAAS_PASSWORD = configure_ipmi_user(IPMI_MAAS_USER) ++ IPMI_MAAS_PASSWORD = configure_ipmi_user_with_backoff(IPMI_MAAS_USER) + + # Attempt to enable IPMI Over Lan. If it is disabled, MAAS won't + # be able to remotely communicate to the BMC. diff --git a/images/maas-region-controller/Dockerfile b/images/maas-region-controller/Dockerfile index 9e92a26..0643a1a 100644 --- a/images/maas-region-controller/Dockerfile +++ b/images/maas-region-controller/Dockerfile @@ -72,11 +72,15 @@ COPY 2.3_mac_address.patch /tmp/2.3_mac_address.patch # sh8121att: allow all requests via the proxy to allow it to work # behind ingress COPY 2.3_proxy_acl.patch /tmp/2.3_proxy_acl.patch +# Patch to add retrying to MaaS BMC user setup, and improve exception handling +copy 2.3_configure_ipmi_user.patch /tmp/2.3_configure_ipmi_user.patch + RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.3_route.patch RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.3_kernel_package.patch RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.3_bios_grub_partition.patch RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_storage.py < /tmp/2.3_bios_grub_preseed.patch RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_enlist.sh < /tmp/2.3_maas_enlist.patch +RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_ipmi_autodetect.py < /tmp/2.3_configure_ipmi_user.patch RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.3_proxy_acl.patch