diff --git a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py index 13188ecb8..7b3dad4d4 100755 --- a/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py +++ b/src/metadataserver/user_data/templates/snippets/maas_ipmi_autodetect.py @@ -235,8 +235,30 @@ def make_ipmi_user_settings(username, password): return user_settings +def configure_ipmi_user_with_backoff(username): + """Create/configure an IPMI user, but with several tries""" + attempt = 1 + max_attempts = 5 + backoff_amount = 30 + while attempt <= max_attempts: + password = None + try: + password = configure_ipmi_user(username) + except: + if (attempt + 1) > max_attempts: + # This is our last attempt, don't catch anything: + raise + + if password is None: + time.sleep(attempt * backoff_amount) + else: + return password + attempt += 1 + + def configure_ipmi_user(username): """Create or configure an IPMI user for remote use.""" + exceptions_caught = [] for password in [ generate_random_password(), generate_random_password(with_special_chars=True), @@ -245,9 +267,11 @@ def configure_ipmi_user(username): try: apply_ipmi_user_settings(user_settings) return password - except subprocess.CalledProcessError: - pass - raise IPMIError("Unable to set BMC password.") + except subprocess.CalledProcessError as e: + exceptions_caught.append(e) + raise IPMIError( + "Unable to set BMC password:\n{}".format(exceptions_caught) + ) def set_ipmi_lan_channel_settings(): @@ -389,7 +413,7 @@ def main(): IPMI_MAAS_USER = args.maas_ipmi_user IPMI_MAAS_PASSWORD = None - IPMI_MAAS_PASSWORD = configure_ipmi_user(IPMI_MAAS_USER) + IPMI_MAAS_PASSWORD = configure_ipmi_user_with_backoff(IPMI_MAAS_USER) # Attempt to enable IPMI Over Lan. If it is disabled, MAAS won't # be able to remotely communicate to the BMC.