Add retries to MaaS BMC user configuration

It has been observed that MaaS will fail to enlist/commission/deploy
nodes if it fails to set up its own user in the BMC during cloud
init. This patch set adds a git patch file to update the MaaS source
code in order to retry setting up the MaaS BMC user if it fails.

This patch set also adds to the exception message sent when MaaS
fails to set up a BMC user.

Change-Id: I475988875acffac620302fae3eed8d236a5a46f7
This commit is contained in:
Carter, Matt (mc981n) 2019-09-16 14:20:20 -05:00
parent e7046aa956
commit 48df9fd6f5
2 changed files with 61 additions and 0 deletions

View File

@ -0,0 +1,57 @@
diff --git a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
index e2c3ce5..7370963 100644
--- a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
+++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py
@@ -229,17 +229,40 @@ def make_ipmi_user_settings(username, password):
return user_settings
+def configure_ipmi_user_with_backoff(username):
+ """Create/configure an IPMI user, but with several tries"""
+ attempt = 1
+ max_attempts = 5
+ backoff_amount = 30
+ while attempt <= max_attempts:
+ password = None
+ try:
+ password = configure_ipmi_user(username)
+ except:
+ if (attempt + 1) > max_attempts:
+ # This is our last attempt, don't catch anything:
+ raise
+
+ if password is None:
+ time.sleep(attempt * backoff_amount)
+ else:
+ return password
+ attempt += 1
+
+
def configure_ipmi_user(username):
"""Create or configure an IPMI user for remote use."""
+ exceptions_caught = []
for password in [generate_random_password(),
generate_random_password(with_special_chars=True)]:
user_settings = make_ipmi_user_settings(username, password)
try:
apply_ipmi_user_settings(user_settings)
return password
- except subprocess.CalledProcessError:
- pass
- raise IPMIError("Unable to set BMC password.")
+ except subprocess.CalledProcessError as e:
+ exceptions_caught.append(e)
+ raise IPMIError(
+ "Unable to set BMC password:\n{}".format(exceptions_caught))
def set_ipmi_lan_channel_settings():
@@ -357,7 +380,7 @@ def main():
IPMI_MAAS_USER = "maas"
IPMI_MAAS_PASSWORD = None
- IPMI_MAAS_PASSWORD = configure_ipmi_user(IPMI_MAAS_USER)
+ IPMI_MAAS_PASSWORD = configure_ipmi_user_with_backoff(IPMI_MAAS_USER)
# Attempt to enable IPMI Over Lan. If it is disabled, MAAS won't
# be able to remotely communicate to the BMC.

View File

@ -72,11 +72,15 @@ COPY 2.3_mac_address.patch /tmp/2.3_mac_address.patch
# sh8121att: allow all requests via the proxy to allow it to work # sh8121att: allow all requests via the proxy to allow it to work
# behind ingress # behind ingress
COPY 2.3_proxy_acl.patch /tmp/2.3_proxy_acl.patch COPY 2.3_proxy_acl.patch /tmp/2.3_proxy_acl.patch
# Patch to add retrying to MaaS BMC user setup, and improve exception handling
copy 2.3_configure_ipmi_user.patch /tmp/2.3_configure_ipmi_user.patch
RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.3_route.patch RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_network.py < /tmp/2.3_route.patch
RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.3_kernel_package.patch RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed.py < /tmp/2.3_kernel_package.patch
RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.3_bios_grub_partition.patch RUN cd /usr/lib/python3/dist-packages/maasserver/models && patch partition.py < /tmp/2.3_bios_grub_partition.patch
RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_storage.py < /tmp/2.3_bios_grub_preseed.patch RUN cd /usr/lib/python3/dist-packages/maasserver && patch preseed_storage.py < /tmp/2.3_bios_grub_preseed.patch
RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_enlist.sh < /tmp/2.3_maas_enlist.patch RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_enlist.sh < /tmp/2.3_maas_enlist.patch
RUN cd /usr/lib/python3/dist-packages/metadataserver/user_data/templates/snippets && patch maas_ipmi_autodetect.py < /tmp/2.3_configure_ipmi_user.patch
RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch
RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch ipaddr.py < /tmp/2.3_mac_address.patch
RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.3_proxy_acl.patch RUN cd /usr/lib/python3/dist-packages/provisioningserver/templates/proxy && patch maas-proxy.conf.template < /tmp/2.3_proxy_acl.patch