Workaround MAAS race condition

- If two threads concurrently update the power parameters
  of VMs on the same libvirt host, it seems to leave MAAS
  with broken state. Add locking in Drydock so that Drydock
  does not do this concurrently.

Change-Id: I85575f4ba48152b4dc79c646871f33b69f845ab9
This commit is contained in:
Scott Hussey 2018-12-17 11:29:52 -06:00
parent 485f919822
commit 54ea0e1374
1 changed files with 36 additions and 31 deletions

View File

@ -15,6 +15,8 @@
import logging
import base64
from threading import Lock, Condition
import drydock_provisioner.error as errors
import drydock_provisioner.drivers.node.maasdriver.models.base as model_base
import drydock_provisioner.drivers.node.maasdriver.models.interface as maas_interface
@ -26,6 +28,8 @@ from bson import BSON
LOG = logging.getLogger(__name__)
power_lock = Lock()
power_cv = Condition(lock=power_lock)
class Machine(model_base.ResourceBase):
@ -388,6 +392,7 @@ class Machine(model_base.ResourceBase):
:param kwargs: Each kwargs key will be prepended with 'power_parameters_' and
added to the list of updates for the node.
"""
with power_cv:
if not power_type:
raise errors.DriverError(
"Cannot set power parameters. Must specify a power type.")
@ -422,7 +427,7 @@ class Machine(model_base.ResourceBase):
Only available after the node has been added to MAAS.
"""
with power_cv:
url = self.interpolate_url()
self.logger.debug("Resetting node power type for machine {}".format(