From 3ca7978bc79fe517714cf9ce76b35cdb4f49f0e8 Mon Sep 17 00:00:00 2001 From: "DeJaeger, Darren (dd118r)" Date: Tue, 13 Jul 2021 10:12:58 -0400 Subject: [PATCH] Drydock request factory tuning This patchset tunes the Drydock MAAS request factory to: a) Implement retries for requests toward MAAS_URL/api/2.0/ b) Bumps the request timeout slightly In addition, restricts threaded actions towards nodes to using the same MAAS client, effectively rate limiting calls to the MAAS api. Change-Id: I2e66105ae332adaed62c9c3bc8cddc63e1f7bf23 --- .../drivers/node/maasdriver/api_client.py | 14 +++++++++++++- .../drivers/node/maasdriver/driver.py | 6 +++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/python/drydock_provisioner/drivers/node/maasdriver/api_client.py b/python/drydock_provisioner/drivers/node/maasdriver/api_client.py index 8f56dc8b..cdc5a190 100644 --- a/python/drydock_provisioner/drivers/node/maasdriver/api_client.py +++ b/python/drydock_provisioner/drivers/node/maasdriver/api_client.py @@ -17,6 +17,8 @@ import logging from oauthlib import oauth1 import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry import requests.auth as req_auth import base64 @@ -58,8 +60,18 @@ class MaasRequestFactory(object): self.base_url = base_url + "/api/2.0/" self.apikey = apikey + # Adapter for maas for request retries + retry_strategy = Retry( + total=3, + status_forcelist=[429, 500, 502, 503, 504], + method_whitelist=["HEAD", "GET", "POST", "PUT", "DELETE", + "OPTIONS", "TRACE"] + ) + self.maas_adapter = HTTPAdapter(max_retries=retry_strategy) + self.signer = MaasOauth(apikey) self.http_session = requests.Session() + self.http_session.mount(self.base_url, self.maas_adapter) # TODO(sh8121att) Get logger name from config self.logger = logging.getLogger('drydock') @@ -160,7 +172,7 @@ class MaasRequestFactory(object): # TODO(sh8121att) timeouts should be configurable timeout = kwargs.pop('timeout', None) if timeout is None: - timeout = (2, 30) + timeout = (5, 60) request = requests.Request( method=method, diff --git a/python/drydock_provisioner/drivers/node/maasdriver/driver.py b/python/drydock_provisioner/drivers/node/maasdriver/driver.py index e7b5b3a8..6ea8a08f 100644 --- a/python/drydock_provisioner/drivers/node/maasdriver/driver.py +++ b/python/drydock_provisioner/drivers/node/maasdriver/driver.py @@ -151,10 +151,10 @@ class MaasNodeDriver(NodeDriver): with concurrent.futures.ThreadPoolExecutor(max_workers=16) as e: subtask_futures = dict() + maas_client = MaasRequestFactory( + config.config_mgr.conf.maasdriver.maas_api_url, + config.config_mgr.conf.maasdriver.maas_api_key) for n in target_nodes: - maas_client = MaasRequestFactory( - config.config_mgr.conf.maasdriver.maas_api_url, - config.config_mgr.conf.maasdriver.maas_api_key) nf = self.orchestrator.create_nodefilter_from_nodelist([n]) subtask = self.orchestrator.create_task( design_ref=task.design_ref,