summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorScott Hussey <sh8121@att.com>2018-12-17 11:29:52 -0600
committerScott Hussey <sh8121@att.com>2018-12-17 11:29:52 -0600
commit54ea0e1374ddd18a5195edc418b5c0b042666f45 (patch)
treeee3cee350089bae5fc8b85daaca3fb5e0f9f9367
parent485f919822c7e425741a19ab507f83ea075529c5 (diff)
Workaround MAAS race condition
- If two threads concurrently update the power parameters of VMs on the same libvirt host, it seems to leave MAAS with broken state. Add locking in Drydock so that Drydock does not do this concurrently. Change-Id: I85575f4ba48152b4dc79c646871f33b69f845ab9
Notes
Notes (review): Code-Review+2: Kaspars Skels <kaspars.skels@gmail.com> Code-Review+2: Craig Anderson <craig.anderson@att.com> Workflow+1: Craig Anderson <craig.anderson@att.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Wed, 19 Dec 2018 20:54:08 +0000 Reviewed-on: https://review.openstack.org/625674 Project: openstack/airship-drydock Branch: refs/heads/master
-rw-r--r--python/drydock_provisioner/drivers/node/maasdriver/models/machine.py67
1 files changed, 36 insertions, 31 deletions
diff --git a/python/drydock_provisioner/drivers/node/maasdriver/models/machine.py b/python/drydock_provisioner/drivers/node/maasdriver/models/machine.py
index aa1d6b7..5186bd1 100644
--- a/python/drydock_provisioner/drivers/node/maasdriver/models/machine.py
+++ b/python/drydock_provisioner/drivers/node/maasdriver/models/machine.py
@@ -15,6 +15,8 @@
15import logging 15import logging
16import base64 16import base64
17 17
18from threading import Lock, Condition
19
18import drydock_provisioner.error as errors 20import drydock_provisioner.error as errors
19import drydock_provisioner.drivers.node.maasdriver.models.base as model_base 21import drydock_provisioner.drivers.node.maasdriver.models.base as model_base
20import drydock_provisioner.drivers.node.maasdriver.models.interface as maas_interface 22import drydock_provisioner.drivers.node.maasdriver.models.interface as maas_interface
@@ -26,6 +28,8 @@ from bson import BSON
26 28
27LOG = logging.getLogger(__name__) 29LOG = logging.getLogger(__name__)
28 30
31power_lock = Lock()
32power_cv = Condition(lock=power_lock)
29 33
30class Machine(model_base.ResourceBase): 34class Machine(model_base.ResourceBase):
31 35
@@ -388,32 +392,33 @@ class Machine(model_base.ResourceBase):
388 :param kwargs: Each kwargs key will be prepended with 'power_parameters_' and 392 :param kwargs: Each kwargs key will be prepended with 'power_parameters_' and
389 added to the list of updates for the node. 393 added to the list of updates for the node.
390 """ 394 """
391 if not power_type: 395 with power_cv:
392 raise errors.DriverError( 396 if not power_type:
393 "Cannot set power parameters. Must specify a power type.") 397 raise errors.DriverError(
398 "Cannot set power parameters. Must specify a power type.")
394 399
395 url = self.interpolate_url() 400 url = self.interpolate_url()
396 401
397 if kwargs: 402 if kwargs:
398 power_params = dict() 403 power_params = dict()
399 404
400 self.logger.debug("Setting node power type to %s." % power_type) 405 self.logger.debug("Setting node power type to %s." % power_type)
401 self.power_type = power_type 406 self.power_type = power_type
402 power_params['power_type'] = power_type 407 power_params['power_type'] = power_type
403 408
404 for k, v in kwargs.items(): 409 for k, v in kwargs.items():
405 power_params['power_parameters_' + k] = v 410 power_params['power_parameters_' + k] = v
406 411
407 self.logger.debug("Updating node %s power parameters: %s" % 412 self.logger.debug("Updating node %s power parameters: %s" %
408 (self.hostname, str(power_params))) 413 (self.hostname, str(power_params)))
409 resp = self.api_client.put(url, files=power_params) 414 resp = self.api_client.put(url, files=power_params)
410 415
411 if resp.status_code == 200: 416 if resp.status_code == 200:
412 return True 417 return True
413 418
414 raise errors.DriverError( 419 raise errors.DriverError(
415 "Failed updating power parameters MAAS url %s - return code %s\n%s" 420 "Failed updating power parameters MAAS url %s - return code %s\n%s"
416 % (url, resp.status_code.resp.text)) 421 % (url, resp.status_code.resp.text))
417 422
418 def reset_power_parameters(self): 423 def reset_power_parameters(self):
419 """Reset power type and parameters for this node to manual. 424 """Reset power type and parameters for this node to manual.
@@ -422,21 +427,21 @@ class Machine(model_base.ResourceBase):
422 427
423 Only available after the node has been added to MAAS. 428 Only available after the node has been added to MAAS.
424 """ 429 """
430 with power_cv:
431 url = self.interpolate_url()
425 432
426 url = self.interpolate_url() 433 self.logger.debug("Resetting node power type for machine {}".format(
427 434 self.resource_id))
428 self.logger.debug("Resetting node power type for machine {}".format( 435 self.power_type = 'manual'
429 self.resource_id)) 436 power_params = {'power_type': 'manual'}
430 self.power_type = 'manual' 437 resp = self.api_client.put(url, files=power_params)
431 power_params = {'power_type': 'manual'}
432 resp = self.api_client.put(url, files=power_params)
433 438
434 if resp.status_code == 200: 439 if resp.status_code == 200:
435 return True 440 return True
436 441
437 raise errors.DriverError( 442 raise errors.DriverError(
438 "Failed updating power parameters MAAS url {} - return code {}\n{}" 443 "Failed updating power parameters MAAS url {} - return code {}\n{}"
439 .format(url, resp.status_code.resp.text)) 444 .format(url, resp.status_code.resp.text))
440 445
441 def update_identity(self, n, domain="local"): 446 def update_identity(self, n, domain="local"):
442 """Update this node's identity based on the Node object ``n`` 447 """Update this node's identity based on the Node object ``n``