Update node-labels through Kubernetes Provisioner

Blueprint: https://airshipit.readthedocs.io/projects/specs/en/latest/specs/approved/k8s_update_node_labels_workflow.html

This commit adds:

1. A new task action ''relabel_nodes'' added to update
nodes labels
2.A new Kubernetes driver added for Kubernetes cluster
interaction through Promenade.

Change-Id: I37c2d7bfda4966d907556036bc2b343df451994c

docs/source/_static/drydock.conf.sample

@@ -276,6 +276,9 @@
 # Logger name for Node driver logging (string value)
 #nodedriver_logger_name = ${global_logger_name}.nodedriver
 
+# Logger name for Kubernetes driver logging (string value)
+#kubernetesdriver_logger_name = ${global_logger_name}.kubernetesdriver
+
 # Logger name for API server logging (string value)
 #control_logger_name = ${global_logger_name}.control
 
@@ -350,6 +353,9 @@
 # Module path string of the Node driver to enable (string value)
 #node_driver = drydock_provisioner.drivers.node.maasdriver.driver.MaasNodeDriver
 
+# Module path string of the Kubernetes driver to enable (string value)
+#kubernetes_driver = drydock_provisioner.drivers.kubernetes.promenade_driver.driver.PromenadeDriver
+
 # Module path string of the Network driver enable (string value)
 #network_driver = <None>
 
@@ -398,6 +404,9 @@
 # Timeout in minutes for deploying a node (integer value)
 #deploy_node = 45
 
+# Timeout in minutes for relabeling a node (integer value)
+#relabel_node = 5
+
 # Timeout in minutes between deployment completion and the all boot actions
 # reporting status (integer value)
 #bootaction_final_status = 15

docs/source/_static/policy.yaml.sample

@@ -38,6 +38,10 @@
 # POST  /api/v1.0/tasks
 #"physical_provisioner:destroy_nodes": "role:admin"
 
+# Create relabel_nodes task
+# POST  /api/v1.0/tasks
+#"physical_provisioner:relabel_nodes": "role:admin"
+
 # Read build data for a node
 # GET  /api/v1.0/nodes/{nodename}/builddata
 #"physical_provisioner:read_build_data": "role:admin"

docs/source/task.rst

@@ -14,7 +14,7 @@ Task Document Schema
 This document can be posted to the Drydock :ref:`tasks-api` to create a new task.::
 
     {
-      "action": "validate_design|verify_site|prepare_site|verify_node|prepare_node|deploy_node|destroy_node",
+      "action": "validate_design|verify_site|prepare_site|verify_node|prepare_node|deploy_node|destroy_node|relabel_nodes",
       "design_ref": "http_uri|deckhand_uri|file_uri",
       "node_filter": {
         "filter_set_type": "intersection|union",
@@ -90,7 +90,7 @@ When querying the state of an existing task, the below document will be returned
       "Kind": "Task",
       "apiVersion": "v1.0",
       "task_id": "uuid",
-      "action": "validate_design|verify_site|prepare_site|verify_node|prepare_node|deploy_node|destroy_node",
+      "action": "validate_design|verify_site|prepare_site|verify_node|prepare_node|deploy_node|destroy_node|relabel_nodes",
       "design_ref": "http_uri|deckhand_uri|file_uri",
       "parent_task_id": "uuid",
       "subtask_id_list": ["uuid","uuid",...],

python/drydock_provisioner/config.py

@@ -81,6 +81,10 @@ class DrydockConfig(object):
             'nodedriver_logger_name',
             default='${global_logger_name}.nodedriver',
             help='Logger name for Node driver logging'),
+        cfg.StrOpt(
+            'kubernetesdriver_logger_name',
+            default='${global_logger_name}.kubernetesdriver',
+            help='Logger name for Kubernetes driver logging'),
         cfg.StrOpt(
             'control_logger_name',
             default='${global_logger_name}.control',
@@ -166,6 +170,11 @@ class DrydockConfig(object):
             default=
             'drydock_provisioner.drivers.node.maasdriver.driver.MaasNodeDriver',
             help='Module path string of the Node driver to enable'),
+        cfg.StrOpt(
+            'kubernetes_driver',
+            default=
+            'drydock_provisioner.drivers.kubernetes.promenade_driver.driver.PromenadeDriver',
+            help='Module path string of the Kubernetes driver to enable'),
         # TODO(sh8121att) Network driver not yet implemented
         cfg.StrOpt(
             'network_driver',
@@ -224,6 +233,10 @@ class DrydockConfig(object):
             default=30,
             help='Timeout in minutes for releasing a node',
         ),
+        cfg.IntOpt(
+            'relabel_node',
+            default=5,
+            help='Timeout in minutes for relabeling a node'),
     ]
 
     def __init__(self):

python/drydock_provisioner/control/tasks.py

@@ -63,6 +63,7 @@ class TasksResource(StatefulResource):
             'prepare_nodes': TasksResource.task_prepare_nodes,
             'deploy_nodes': TasksResource.task_deploy_nodes,
             'destroy_nodes': TasksResource.task_destroy_nodes,
+            'relabel_nodes': TasksResource.task_relabel_nodes,
         }
 
         try:
@@ -253,6 +254,30 @@ class TasksResource(StatefulResource):
             self.return_error(
                 resp, falcon.HTTP_400, message=ex.msg, retry=False)
 
+    @policy.ApiEnforcer('physical_provisioner:relabel_nodes')
+    def task_relabel_nodes(self, req, resp, json_data):
+        """Create async task for relabel nodes."""
+        action = json_data.get('action', None)
+
+        if action != 'relabel_nodes':
+            self.error(
+                req.context,
+                "Task body ended up in wrong handler: action %s in task_relabel_nodes"
+                % action)
+            self.return_error(
+                resp, falcon.HTTP_500, message="Error", retry=False)
+
+        try:
+            task = self.create_task(json_data, req.context)
+            resp.body = json.dumps(task.to_dict())
+            resp.append_header('Location',
+                               "/api/v1.0/tasks/%s" % str(task.task_id))
+            resp.status = falcon.HTTP_201
+        except errors.InvalidFormat as ex:
+            self.error(req.context, ex.msg)
+            self.return_error(
+                resp, falcon.HTTP_400, message=ex.msg, retry=False)
+
     def create_task(self, task_body, req_context):
         """General task creation.
 

python/drydock_provisioner/drivers/kubernetes/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Drivers for use for Kubernetes provisioner interaction."""

python/drydock_provisioner/drivers/kubernetes/driver.py

@@ -0,0 +1,46 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Generic driver for Kubernetes Interaction."""
+
+import drydock_provisioner.objects.fields as hd_fields
+import drydock_provisioner.error as errors
+
+from drydock_provisioner.drivers.driver import ProviderDriver
+
+
+class KubernetesDriver(ProviderDriver):
+
+    driver_name = "Kubernetes_generic"
+    driver_key = "Kubernetes_generic"
+    driver_desc = "Generic Kubernetes Driver"
+
+    def __init__(self, **kwargs):
+        super(KubernetesDriver, self).__init__(**kwargs)
+
+        self.supported_actions = [
+            hd_fields.OrchestratorAction.RelabelNode,
+        ]
+
+    def execute_task(self, task_id):
+        task = self.state_manager.get_task(task_id)
+        task_action = task.action
+
+        if task_action in self.supported_actions:
+            task.success()
+            task.set_status(hd_fields.TaskStatus.Complete)
+            task.save()
+            return
+        else:
+            raise errors.DriverError("Unsupported action %s for driver %s" %
+                                     (task_action, self.driver_desc))

python/drydock_provisioner/drivers/kubernetes/promenade_driver/README.md

@@ -0,0 +1,4 @@
+# Promenade Kubernetes Driver #
+
+This driver will handle the interaction with Promenade for
+any changes applied to Kubernetes cluster nodes.

python/drydock_provisioner/drivers/kubernetes/promenade_driver/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Drivers for use for Promenade interaction."""

python/drydock_provisioner/drivers/kubernetes/promenade_driver/actions/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Kubernetes task driver action for Promenade interaction."""

python/drydock_provisioner/drivers/kubernetes/promenade_driver/actions/k8s_node.py

@@ -0,0 +1,83 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Task driver for Promenade interaction."""
+
+import logging
+
+import drydock_provisioner.error as errors
+import drydock_provisioner.config as config
+import drydock_provisioner.objects.fields as hd_fields
+from drydock_provisioner.orchestrator.actions.orchestrator import BaseAction
+
+
+class PromenadeAction(BaseAction):
+    def __init__(self, *args, prom_client=None):
+        super().__init__(*args)
+
+        self.promenade_client = prom_client
+
+        self.logger = logging.getLogger(
+            config.config_mgr.conf.logging.kubernetesdriver_logger_name)
+
+
+class RelabelNode(PromenadeAction):
+    """Action to relabel kubernetes node."""
+
+    def start(self):
+
+        self.task.set_status(hd_fields.TaskStatus.Running)
+        self.task.save()
+
+        try:
+            site_design = self._load_site_design()
+        except errors.OrchestratorError:
+            self.task.add_status_msg(
+                msg="Error loading site design.",
+                error=True,
+                ctx='NA',
+                ctx_type='NA')
+            self.task.set_status(hd_fields.TaskStatus.Complete)
+            self.task.failure()
+            self.task.save()
+            return
+
+        nodes = self.orchestrator.process_node_filter(self.task.node_filter,
+                                                      site_design)
+
+        for n in nodes:
+            # Relabel node through Promenade
+            try:
+                self.logger.info("Relabeling node %s with node label data." % n.name)
+
+                labels_dict = n.get_node_labels()
+                msg = "Set labels %s for node %s" % (str(labels_dict), n.name)
+
+                self.task.add_status_msg(
+                    msg=msg, error=False, ctx=n.name, ctx_type='node')
+
+                # Call promenade to invoke relabel node
+                self.promenade_client.relabel_node(n.get_id(), labels_dict)
+                self.task.success(focus=n.get_id())
+            except Exception as ex:
+                msg = "Error relabeling node %s with label data" % n.name
+                self.logger.warning(msg + ": " + str(ex))
+                self.task.failure(focus=n.get_id())
+                self.task.add_status_msg(
+                    msg=msg, error=True, ctx=n.name, ctx_type='node')
+                continue
+
+        self.task.set_status(hd_fields.TaskStatus.Complete)
+        self.task.save()
+
+        return

python/drydock_provisioner/drivers/kubernetes/promenade_driver/driver.py

@@ -0,0 +1,156 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Task driver for Promenade"""
+
+import logging
+import uuid
+import concurrent.futures
+
+from oslo_config import cfg
+
+import drydock_provisioner.error as errors
+import drydock_provisioner.objects.fields as hd_fields
+import drydock_provisioner.config as config
+from drydock_provisioner.drivers.kubernetes.driver import KubernetesDriver
+from drydock_provisioner.drivers.kubernetes.promenade_driver.promenade_client \
+    import PromenadeClient
+
+from .actions.k8s_node import RelabelNode
+
+
+class PromenadeDriver(KubernetesDriver):
+
+    driver_name = 'promenadedriver'
+    driver_key = 'promenadedriver'
+    driver_desc = 'Promenade Kubernetes Driver'
+
+    action_class_map = {
+        hd_fields.OrchestratorAction.RelabelNode:
+        RelabelNode,
+    }
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.logger = logging.getLogger(
+            cfg.CONF.logging.kubernetesdriver_logger_name)
+
+    def execute_task(self, task_id):
+        # actions that should be threaded for execution
+        threaded_actions = [
+            hd_fields.OrchestratorAction.RelabelNode,
+        ]
+
+        action_timeouts = {
+            hd_fields.OrchestratorAction.RelabelNode:
+            config.config_mgr.conf.timeouts.relabel_node,
+        }
+
+        task = self.state_manager.get_task(task_id)
+
+        if task is None:
+            raise errors.DriverError("Invalid task %s" % (task_id))
+
+        if task.action not in self.supported_actions:
+            raise errors.DriverError("Driver %s doesn't support task action %s"
+                                     % (self.driver_desc, task.action))
+
+        task.set_status(hd_fields.TaskStatus.Running)
+        task.save()
+
+        if task.action in threaded_actions:
+            if task.retry > 0:
+                msg = "Retrying task %s on previous failed entities." % str(
+                    task.get_id())
+                task.add_status_msg(
+                    msg=msg,
+                    error=False,
+                    ctx=str(task.get_id()),
+                    ctx_type='task')
+                target_nodes = self.orchestrator.get_target_nodes(
+                    task, failures=True)
+            else:
+                target_nodes = self.orchestrator.get_target_nodes(task)
+
+            with concurrent.futures.ThreadPoolExecutor() as e:
+                subtask_futures = dict()
+                for n in target_nodes:
+                    prom_client = PromenadeClient()
+                    nf = self.orchestrator.create_nodefilter_from_nodelist([n])
+                    subtask = self.orchestrator.create_task(
+                        design_ref=task.design_ref,
+                        action=task.action,
+                        node_filter=nf,
+                        retry=task.retry)
+                    task.register_subtask(subtask)
+
+                    action = self.action_class_map.get(task.action, None)(
+                        subtask,
+                        self.orchestrator,
+                        self.state_manager,
+                        prom_client=prom_client)
+                    subtask_futures[subtask.get_id().bytes] = e.submit(
+                        action.start)
+
+                timeout = action_timeouts.get(
+                    task.action,
+                    config.config_mgr.conf.timeouts.relabel_node)
+                finished, running = concurrent.futures.wait(
+                    subtask_futures.values(), timeout=(timeout * 60))
+
+            for t, f in subtask_futures.items():
+                if not f.done():
+                    task.add_status_msg(
+                        "Subtask timed out before completing.",
+                        error=True,
+                        ctx=str(uuid.UUID(bytes=t)),
+                        ctx_type='task')
+                    task.failure()
+                else:
+                    if f.exception():
+                        msg = ("Subtask %s raised unexpected exception: %s"
+                               % (str(uuid.UUID(bytes=t)), str(f.exception())))
+                        self.logger.error(msg, exc_info=f.exception())
+                        task.add_status_msg(
+                            msg=msg,
+                            error=True,
+                            ctx=str(uuid.UUID(bytes=t)),
+                            ctx_type='task')
+                        task.failure()
+
+            task.bubble_results()
+            task.align_result()
+        else:
+            try:
+                prom_client = PromenadeClient()
+                action = self.action_class_map.get(task.action, None)(
+                    task,
+                    self.orchestrator,
+                    self.state_manager,
+                    prom_client=prom_client)
+                action.start()
+            except Exception as e:
+                msg = ("Subtask for action %s raised unexpected exception: %s"
+                       % (task.action, str(e)))
+                self.logger.error(msg, exc_info=e)
+                task.add_status_msg(
+                    msg=msg,
+                    error=True,
+                    ctx=str(task.get_id()),
+                    ctx_type='task')
+                task.failure()
+
+        task.set_status(hd_fields.TaskStatus.Complete)
+        task.save()
+
+        return

python/drydock_provisioner/drivers/kubernetes/promenade_driver/promenade_client.py

@@ -0,0 +1,296 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Client for submitting authenticated requests to Promenade API."""
+
+import logging
+import requests
+from urllib.parse import urlparse
+
+from keystoneauth1 import exceptions as exc
+
+import drydock_provisioner.error as errors
+from drydock_provisioner.util import KeystoneUtils
+
+# TODO: Remove this local implementation of Promenade Session and client once
+# Promenade api client is available as part of Promenade project.
+class PromenadeSession(object):
+    """
+    A session to the Promenade API maintaining credentials and API options
+
+    :param string marker: (optional) external context marker
+    :param tuple timeout: (optional) a tuple of connect, read timeout values
+        to use as the default for invocations using this session. A single
+        value may also be supplied instead of a tuple to indicate only the
+        read timeout to use
+    """
+
+    def __init__(self,
+                 scheme='http',
+                 marker=None,
+                 timeout=None):
+        self.logger = logging.getLogger(__name__)
+        self.__session = requests.Session()
+
+        self.set_auth()
+
+        self.marker = marker
+        self.__session.headers.update({'X-Context-Marker': marker})
+
+        self.prom_url = self._get_prom_url()
+        self.port = self.prom_url.port
+        self.host = self.prom_url.hostname
+        self.scheme = scheme
+
+        if self.port:
+            self.base_url = "%s://%s:%s/api/" % (self.scheme, self.host,
+                                                 self.port)
+        else:
+            # assume default port for scheme
+            self.base_url = "%s://%s/api/" % (self.scheme, self.host)
+
+        self.default_timeout = self._calc_timeout_tuple((20, 30), timeout)
+
+    def set_auth(self):
+
+            auth_header = self._auth_gen()
+            self.__session.headers.update(auth_header)
+
+    def get(self, route, query=None, timeout=None):
+        """
+        Send a GET request to Promenade.
+
+        :param string route: The URL string following the hostname and API prefix
+        :param dict query: A dict of k, v pairs to add to the query string
+        :param timeout: A single or tuple value for connect, read timeout.
+            A single value indicates the read timeout only
+        :return: A requests.Response object
+        """
+        auth_refresh = False
+        while True:
+            url = self.base_url + route
+            self.logger.debug('GET ' + url)
+            self.logger.debug('Query Params: ' + str(query))
+            resp = self.__session.get(
+                url, params=query, timeout=self._timeout(timeout))
+
+            if resp.status_code == 401 and not auth_refresh:
+                self.set_auth()
+                auth_refresh = True
+            else:
+                break
+
+        return resp
+
+    def put(self, endpoint, query=None, body=None, data=None, timeout=None):
+        """
+        Send a PUT request to Promenade. If both body and data are specified,
+        body will be used.
+
+        :param string endpoint: The URL string following the hostname and API prefix
+        :param dict query: A dict of k, v parameters to add to the query string
+        :param string body: A string to use as the request body. Will be treated as raw
+        :param data: Something json.dumps(s) can serialize. Result will be used as the request body
+        :param timeout: A single or tuple value for connect, read timeout.
+            A single value indicates the read timeout only
+        :return: A requests.Response object
+        """
+        auth_refresh = False
+        url = self.base_url + endpoint
+        while True:
+            self.logger.debug('PUT ' + url)
+            self.logger.debug('Query Params: ' + str(query))
+            if body is not None:
+                self.logger.debug(
+                    "Sending PUT with explicit body: \n%s" % body)
+                resp = self.__session.put(
+                    self.base_url + endpoint,
+                    params=query,
+                    data=body,
+                    timeout=self._timeout(timeout))
+            else:
+                self.logger.debug(
+                    "Sending PUT with JSON body: \n%s" % str(data))
+                resp = self.__session.put(
+                    self.base_url + endpoint,
+                    params=query,
+                    json=data,
+                    timeout=self._timeout(timeout))
+            if resp.status_code == 401 and not auth_refresh:
+                self.set_auth()
+                auth_refresh = True
+            else:
+                break
+
+        return resp
+
+    def post(self, endpoint, query=None, body=None, data=None, timeout=None):
+        """
+        Send a POST request to Drydock. If both body and data are specified,
+        body will be used.
+
+        :param string endpoint: The URL string following the hostname and API prefix
+        :param dict query: A dict of k, v parameters to add to the query string
+        :param string body: A string to use as the request body. Will be treated as raw
+        :param data: Something json.dumps(s) can serialize. Result will be used as the request body
+        :param timeout: A single or tuple value for connect, read timeout.
+            A single value indicates the read timeout only
+        :return: A requests.Response object
+        """
+        auth_refresh = False
+        url = self.base_url + endpoint
+        while True:
+            self.logger.debug('POST ' + url)
+            self.logger.debug('Query Params: ' + str(query))
+            if body is not None:
+                self.logger.debug(
+                    "Sending POST with explicit body: \n%s" % body)
+                resp = self.__session.post(
+                    self.base_url + endpoint,
+                    params=query,
+                    data=body,
+                    timeout=self._timeout(timeout))
+            else:
+                self.logger.debug(
+                    "Sending POST with JSON body: \n%s" % str(data))
+                resp = self.__session.post(
+                    self.base_url + endpoint,
+                    params=query,
+                    json=data,
+                    timeout=self._timeout(timeout))
+            if resp.status_code == 401 and not auth_refresh:
+                self.set_auth()
+                auth_refresh = True
+            else:
+                break
+
+        return resp
+
+    def _timeout(self, timeout=None):
+        """Calculate the default timeouts for this session
+
+        :param timeout: A single or tuple value for connect, read timeout.
+            A single value indicates the read timeout only
+        :return: the tuple of the default timeouts used for this session
+        """
+        return self._calc_timeout_tuple(self.default_timeout, timeout)
+
+    def _calc_timeout_tuple(self, def_timeout, timeout=None):
+        """Calculate the default timeouts for this session
+
+        :param def_timeout: The default timeout tuple to be used if no specific
+            timeout value is supplied
+        :param timeout: A single or tuple value for connect, read timeout.
+            A single value indicates the read timeout only
+        :return: the tuple of the timeouts calculated
+        """
+        connect_timeout, read_timeout = def_timeout
+
+        try:
+            if isinstance(timeout, tuple):
+                if all(isinstance(v, int)
+                       for v in timeout) and len(timeout) == 2:
+                    connect_timeout, read_timeout = timeout
+                else:
+                    raise ValueError("Tuple non-integer or wrong length")
+            elif isinstance(timeout, int):
+                read_timeout = timeout
+            elif timeout is not None:
+                raise ValueError("Non integer timeout value")
+        except ValueError:
+            self.logger.warn(
+                "Timeout value must be a tuple of integers or a "
+                "single integer. Proceeding with values of "
+                "(%s, %s)", connect_timeout, read_timeout)
+        return (connect_timeout, read_timeout)
+
+    def _get_ks_session(self):
+        # Get keystone session object
+
+        try:
+            ks_session = KeystoneUtils.get_session()
+        except exc.AuthorizationFailure as aferr:
+            self.logger.error(
+                'Could not authorize against Keystone: %s',
+                str(aferr))
+            raise errors.DriverError('Could not authorize against Keystone: %s',
+                                     str(aferr))
+
+        return ks_session
+
+    def _get_prom_url(self):
+        # Get promenade url from Keystone session object
+
+        ks_session = self._get_ks_session()
+
+        try:
+            prom_endpoint = ks_session.get_endpoint(
+                interface='internal',
+                service_type='kubernetesprovisioner')
+        except exc.EndpointNotFound:
+            self.logger.error("Could not find an internal interface"
+                              " defined in Keystone for Promenade")
+
+            raise errors.DriverError("Could not find an internal interface"
+                                     " defined in Keystone for Promenade")
+
+        prom_url = urlparse(prom_endpoint)
+
+        return prom_url
+
+    def _auth_gen(self):
+        # Get auth token from Keystone session
+        token = self._get_ks_session().get_auth_headers().get('X-Auth-Token')
+        return [('X-Auth-Token', token)]
+
+
+class PromenadeClient(object):
+    """"
+    A client for the Promenade API
+    """
+
+    def __init__(self):
+        self.session = PromenadeSession()
+        self.logger = logging.getLogger(__name__)
+
+    def relabel_node(self, node_id, node_labels):
+        """ Relabel kubernetes node
+
+        :param string node_id: Node id for node to be relabeled.
+        :param dict node_labels: The dictionary representation of node labels
+                                 that needs be re-applied to the node.
+        :return: response
+        """
+
+        route = 'v1.0/node-labels/{}'.format(node_id)
+
+        self.logger.debug("promenade_client is calling %s API: body is %s" %
+                          (route, str(node_labels)))
+
+        resp = self.session.put(route, data=node_labels)
+
+        self._check_response(resp)
+
+        return resp.json()
+
+    def _check_response(self, resp):
+        if resp.status_code == 401:
+            raise errors.ClientUnauthorizedError(
+                "Unauthorized access to %s, include valid token." % resp.url)
+        elif resp.status_code == 403:
+            raise errors.ClientForbiddenError(
+                "Forbidden access to %s" % resp.url)
+        elif not resp.ok:
+            raise errors.ClientError(
+                "Error - received %d: %s" % (resp.status_code, resp.text),
+                code=resp.status_code)

python/drydock_provisioner/objects/fields.py

@@ -31,6 +31,7 @@ class OrchestratorAction(BaseDrydockEnum):
     DeployNodes = 'deploy_nodes'
     DestroyNodes = 'destroy_nodes'
     BootactionReport = 'bootaction_report'
+    RelabelNodes = 'relabel_nodes'
 
     # OOB driver actions
     ValidateOobServices = 'validate_oob_services'
@@ -64,14 +65,17 @@ class OrchestratorAction(BaseDrydockEnum):
     ConfigurePortProvisioning = 'config_port_provisioning'
     ConfigurePortProduction = 'config_port_production'
 
+    # Kubernetes driver actions
+    RelabelNode = 'relabel_node'
+
     ALL = (Noop, ValidateDesign, VerifySite, PrepareSite, VerifyNodes,
            PrepareNodes, DeployNodes, BootactionReport, DestroyNodes,
-           ConfigNodePxe, SetNodeBoot, PowerOffNode, PowerOnNode,
+           RelabelNodes, ConfigNodePxe, SetNodeBoot, PowerOffNode,
-           PowerCycleNode, InterrogateOob, CreateNetworkTemplate,
+           PowerOnNode, PowerCycleNode, InterrogateOob, RelabelNode,
-           CreateStorageTemplate, CreateBootMedia, PrepareHardwareConfig,
+           CreateNetworkTemplate, CreateStorageTemplate, CreateBootMedia,
-           ConfigureHardware, InterrogateNode, ApplyNodeNetworking,
+           PrepareHardwareConfig, ConfigureHardware, InterrogateNode,
-           ApplyNodeStorage, ApplyNodePlatform, DeployNode, DestroyNode,
+           ApplyNodeNetworking, ApplyNodeStorage, ApplyNodePlatform,
-           ConfigureNodeProvisioner)
+           DeployNode, DestroyNode, ConfigureNodeProvisioner)
 
 
 class OrchestratorActionField(fields.BaseEnumField):

python/drydock_provisioner/objects/node.py

@@ -326,6 +326,17 @@ class BaremetalNode(drydock_provisioner.objects.hostprofile.HostProfile):
                 alias)
             return alias
 
+    def get_node_labels(self):
+        """Get node labels.
+        """
+
+        labels_dict = {}
+        for k, v in self.owner_data.items():
+            labels_dict[k] = v
+        self.logger.debug("node labels data : %s." % str(labels_dict))
+        # TODO: Generate node labels
+
+        return labels_dict
 
 @base.DrydockObjectRegistry.register
 class BaremetalNodeList(base.DrydockObjectListBase, base.DrydockObject):

python/drydock_provisioner/orchestrator/actions/orchestrator.py

@@ -1035,6 +1035,66 @@ class DeployNodes(BaseAction):
         return
 
 
+class RelabelNodes(BaseAction):
+    """Action to relabel a node"""
+
+    def start(self):
+        """Start executing this action."""
+        self.task.set_status(hd_fields.TaskStatus.Running)
+        self.task.save()
+
+        kubernetes_driver = self.orchestrator.enabled_drivers['kubernetes']
+
+        if kubernetes_driver is None:
+            self.task.set_status(hd_fields.TaskStatus.Complete)
+            self.task.add_status_msg(
+                msg="No kubernetes driver is enabled, ending task.",
+                error=True,
+                ctx=str(self.task.get_id()),
+                ctx_type='task')
+            self.task.result.set_message("No KubernetesDriver enabled.")
+            self.task.result.set_reason("Bad Configuration.")
+            self.task.failure()
+            self.task.save()
+            return
+
+        target_nodes = self.orchestrator.get_target_nodes(self.task)
+
+        if not target_nodes:
+            self.task.add_status_msg(
+                msg="No nodes in scope, nothing to relabel.",
+                error=False,
+                ctx='NA',
+                ctx_type='NA')
+            self.task.success()
+            self.task.set_status(hd_fields.TaskStatus.Complete)
+            self.task.save()
+            return
+
+        nf = self.orchestrator.create_nodefilter_from_nodelist(target_nodes)
+
+        relabel_node_task = self.orchestrator.create_task(
+            design_ref=self.task.design_ref,
+            action=hd_fields.OrchestratorAction.RelabelNode,
+            node_filter=nf)
+        self.task.register_subtask(relabel_node_task)
+
+        self.logger.info(
+            "Starting kubernetes driver task %s to relabel nodes." %
+            (relabel_node_task.get_id()))
+        kubernetes_driver.execute_task(relabel_node_task.get_id())
+
+        relabel_node_task = self.state_manager.get_task(
+            relabel_node_task.get_id())
+
+        self.task.bubble_results(
+            action_filter=hd_fields.OrchestratorAction.RelabelNode)
+        self.task.align_result()
+
+        self.task.set_status(hd_fields.TaskStatus.Complete)
+        self.task.save()
+
+
 class BootactionReport(BaseAction):
     """Wait for nodes to report status of boot action."""
 

python/drydock_provisioner/orchestrator/orchestrator.py

@@ -33,6 +33,7 @@ from .actions.orchestrator import PrepareSite
 from .actions.orchestrator import VerifyNodes
 from .actions.orchestrator import PrepareNodes
 from .actions.orchestrator import DeployNodes
+from .actions.orchestrator import RelabelNodes
 from .actions.orchestrator import DestroyNodes
 from .validations.validator import Validator
 
@@ -102,6 +103,15 @@ class Orchestrator(object):
                     self.enabled_drivers['network'] = network_driver_class(
                         state_manager=state_manager, orchestrator=self)
 
+            kubernetes_driver_name = enabled_drivers.kubernetes_driver
+            if kubernetes_driver_name is not None:
+                m, c = kubernetes_driver_name.rsplit('.', 1)
+                kubernetes_driver_class = getattr(
+                    importlib.import_module(m), c, None)
+                if kubernetes_driver_class is not None:
+                    self.enabled_drivers['kubernetes'] = kubernetes_driver_class(
+                        state_manager=state_manager, orchestrator=self)
+
     def watch_for_tasks(self):
         """Start polling the database watching for Queued tasks to execute."""
         orch_task_actions = {
@@ -112,6 +122,7 @@ class Orchestrator(object):
             hd_fields.OrchestratorAction.VerifyNodes: VerifyNodes,
             hd_fields.OrchestratorAction.PrepareNodes: PrepareNodes,
             hd_fields.OrchestratorAction.DeployNodes: DeployNodes,
+            hd_fields.OrchestratorAction.RelabelNodes: RelabelNodes,
             hd_fields.OrchestratorAction.DestroyNodes: DestroyNodes,
         }
 

python/drydock_provisioner/orchestrator/readme.md

@@ -111,6 +111,11 @@ success
 
 Destroy current node configuration and rebootstrap from scratch
 
+### RelabelNode ###
+
+Relabel current Kubernetes cluster node through Kubernetes
+provisioner.
+
 ## Integration with Drivers ##
 
 Based on the requested task and the current known state of a node

python/drydock_provisioner/policy.py

@@ -95,6 +95,12 @@ class DrydockPolicy(object):
                                          'path': '/api/v1.0/tasks',
                                          'method': 'POST'
                                      }]),
+        policy.DocumentedRuleDefault('physical_provisioner:relabel_nodes',
+                                     'role:admin', 'Create relabel_nodes task',
+                                     [{
+                                         'path': '/api/v1.0/tasks',
+                                         'method': 'POST'
+                                     }]),
         policy.DocumentedRuleDefault(
             'physical_provisioner:read_build_data', 'role:admin',
             'Read build data for a node',

python/tests/unit/test_k8sdriver_promenade_client.py

@@ -0,0 +1,194 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from unittest import mock
+from urllib.parse import urlparse
+
+import pytest
+import responses
+
+import drydock_provisioner.error as errors
+from drydock_provisioner.drivers.kubernetes.promenade_driver.promenade_client \
+    import PromenadeSession, PromenadeClient
+
+PROM_URL = urlparse('http://promhost:80/api/v1.0')
+PROM_HOST = 'promhost'
+
+
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession._get_prom_url',
+    return_value=PROM_URL)
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession.set_auth',
+    return_value=None)
+@responses.activate
+def test_put(patch1, patch2):
+    """
+    Test put functionality
+    """
+    responses.add(
+        responses.PUT,
+        'http://promhost:80/api/v1.0/node-label/n1',
+        body='{"key1":"label1"}',
+        status=200)
+
+    prom_session = PromenadeSession()
+    result = prom_session.put('v1.0/node-label/n1',
+                              body='{"key1":"label1"}',
+                              timeout=(60, 60))
+
+    assert PROM_HOST == prom_session.host
+    assert result.status_code == 200
+
+
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession._get_prom_url',
+    return_value=PROM_URL)
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession.set_auth',
+    return_value=None)
+@responses.activate
+def test_get(patch1, patch2):
+    """
+    Test get functionality
+    """
+    responses.add(
+        responses.GET,
+        'http://promhost:80/api/v1.0/node-label/n1',
+        status=200)
+
+    prom_session = PromenadeSession()
+    result = prom_session.get('v1.0/node-label/n1',
+                              timeout=(60, 60))
+
+    assert result.status_code == 200
+
+
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession._get_prom_url',
+    return_value=PROM_URL)
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession.set_auth',
+    return_value=None)
+@responses.activate
+def test_post(patch1, patch2):
+    """
+    Test post functionality
+    """
+    responses.add(
+        responses.POST,
+        'http://promhost:80/api/v1.0/node-label/n1',
+        body='{"key1":"label1"}',
+        status=200)
+
+    prom_session = PromenadeSession()
+    result = prom_session.post('v1.0/node-label/n1',
+                               body='{"key1":"label1"}',
+                               timeout=(60, 60))
+
+    assert PROM_HOST == prom_session.host
+    assert result.status_code == 200
+
+
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession._get_prom_url',
+    return_value=PROM_URL)
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession.set_auth',
+    return_value=None)
+@responses.activate
+def test_relabel_node(patch1, patch2):
+    """
+    Test relabel node call from Promenade
+    Client
+    """
+    responses.add(
+        responses.PUT,
+        'http://promhost:80/api/v1.0/node-labels/n1',
+        body='{"key1":"label1"}',
+        status=200)
+
+    prom_client = PromenadeClient()
+
+    result = prom_client.relabel_node('n1', {"key1": "label1"})
+
+    assert result == {"key1": "label1"}
+
+
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession._get_prom_url',
+    return_value=PROM_URL)
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession.set_auth',
+    return_value=None)
+@responses.activate
+def test_relabel_node_403_status(patch1, patch2):
+    """
+    Test relabel node with 403 resp status
+    """
+    responses.add(
+        responses.PUT,
+        'http://promhost:80/api/v1.0/node-labels/n1',
+        body='{"key1":"label1"}',
+        status=403)
+
+    prom_client = PromenadeClient()
+
+    with pytest.raises(errors.ClientForbiddenError):
+        prom_client.relabel_node('n1', {"key1": "label1"})
+
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession._get_prom_url',
+    return_value=PROM_URL)
+@mock.patch(
+    'drydock_provisioner.drivers.kubernetes'
+    '.promenade_driver.promenade_client'
+    '.PromenadeSession.set_auth',
+    return_value=None)
+@responses.activate
+def test_relabel_node_401_status(patch1, patch2):
+    """
+    Test relabel node with 401 resp status
+    """
+    responses.add(
+        responses.PUT,
+        'http://promhost:80/api/v1.0/node-labels/n1',
+        body='{"key1":"label1"}',
+        status=401)
+
+    prom_client = PromenadeClient()
+
+    with pytest.raises(errors.ClientUnauthorizedError):
+        prom_client.relabel_node('n1', {"key1": "label1"})