Promenade validateDesign for Shipyard

SHIPYARD-342: Shipyard Integration with Promenade to Validate Design

Calls Promenade validateDesign API to validate site design.

Change-Id: Ia763983ed9857d4e5b13cfb11d3654e75e6578a4

charts/shipyard/values.yaml

@@ -373,6 +373,8 @@ conf:
       destroy_node_query_interval: 30
       destroy_node_task_timeout: 900
       cluster_join_check_backoff_time: 120
+    promenade:
+      service_type: kubernetesprovisioner
     keystone_authtoken:
       delay_auth_decision: true
       auth_type: password

docs/source/_static/shipyard.conf.sample

@@ -294,6 +294,18 @@
 #named_log_levels = keystoneauth:20,keystonemiddleware:20
 
 
+[promenade]
+
+#
+# From shipyard_api
+#
+
+# The service type for the service playing the role of Promenade. The specified
+# type is used to perform the service lookup in the Keystone service catalog.
+# (string value)
+#service_type = kubernetesprovisioner
+
+
 [requests_config]
 
 #

src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample

@@ -294,6 +294,18 @@
 #named_log_levels = keystoneauth:20,keystonemiddleware:20
 
 
+[promenade]
+
+#
+# From shipyard_api
+#
+
+# The service type for the service playing the role of Promenade. The specified
+# type is used to perform the service lookup in the Keystone service catalog.
+# (string value)
+#service_type = kubernetesprovisioner
+
+
 [requests_config]
 
 #

src/bin/shipyard_airflow/shipyard_airflow/conf/config.py

@@ -89,7 +89,7 @@ SECTIONS = [
     ),
     ConfigSection(
         name='shipyard',
-        title='Shipyard connection info',
+        title='Shipyard Connection Info',
         options=[
             cfg.StrOpt(
                 'service_type',
@@ -104,7 +104,7 @@ SECTIONS = [
     ),
     ConfigSection(
         name='deckhand',
-        title='Deckhand connection info',
+        title='Deckhand Connection Info',
         options=[
             cfg.StrOpt(
                 'service_type',
@@ -119,7 +119,7 @@ SECTIONS = [
     ),
     ConfigSection(
         name='armada',
-        title='Armada connection info',
+        title='Armada Connection Info',
         options=[
             cfg.StrOpt(
                 'service_type',
@@ -134,7 +134,7 @@ SECTIONS = [
     ),
     ConfigSection(
         name='drydock',
-        title='Drydock connection info',
+        title='Drydock Connection Info',
         options=[
             cfg.StrOpt(
                 'service_type',
@@ -147,6 +147,21 @@ SECTIONS = [
             ),
         ]
     ),
+    ConfigSection(
+        name='promenade',
+        title='Promenade Connection Info',
+        options=[
+            cfg.StrOpt(
+                'service_type',
+                default='kubernetesprovisioner',
+                help=(
+                    'The service type for the service playing the role '
+                    'of Promenade. The specified type is used to perform '
+                    'the service lookup in the Keystone service catalog.'
+                )
+            ),
+        ]
+    ),
     ConfigSection(
         name='requests_config',
         title='Requests Configuration',

src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py

@@ -732,6 +732,10 @@ def _get_validation_endpoints():
             'name': 'Armada',
             'url': val_ep.format(get_endpoint(Endpoints.ARMADA))
         },
+        {
+            'name': 'Promenade',
+            'url': val_ep.format(get_endpoint(Endpoints.PROMENADE))
+        }
     ]
 
 

src/bin/shipyard_airflow/shipyard_airflow/control/service_endpoints.py

@@ -39,6 +39,7 @@ class Endpoints(enum.Enum):
     DRYDOCK = 'drydock'
     ARMADA = 'armada'
     DECKHAND = 'deckhand'
+    PROMENADE = 'promenade'
 
 
 def _get_service_type(endpoint):
@@ -57,6 +58,7 @@ def _get_service_type(endpoint):
             Endpoints.DRYDOCK: CONF.drydock.service_type,
             Endpoints.ARMADA: CONF.armada.service_type,
             Endpoints.DECKHAND: CONF.deckhand.service_type,
+            Endpoints.PROMENADE: CONF.promenade.service_type
         }
         return endpoint_values.get(endpoint)
     raise AppError(

src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py

@@ -16,6 +16,7 @@ from airflow.models import DAG
 from airflow.operators import ArmadaValidateDesignOperator
 from airflow.operators import DeckhandValidateSiteDesignOperator
 from airflow.operators import DrydockValidateDesignOperator
+from airflow.operators import PromenadeValidateSiteDesignOperator
 
 from config_path import config_path
 
@@ -52,4 +53,12 @@ def validate_site_design(parent_dag_name, child_dag_name, args):
         retries=3,
         dag=dag)
 
+    promenade_validate_docs = PromenadeValidateSiteDesignOperator(
+        task_id='promenade_validate_site_design',
+        shipyard_conf=config_path,
+        main_dag_name=parent_dag_name,
+        sub_dag_name=child_dag_name,
+        retries=3,
+        dag=dag)
+
     return dag

src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py

@@ -12,14 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
+import os
 
 from airflow.utils.decorators import apply_defaults
 from airflow.plugins_manager import AirflowPlugin
 from airflow.exceptions import AirflowException
 
-from service_endpoint import ucp_service_endpoint
+try:
-from service_token import shipyard_service_token
+    from service_endpoint import ucp_service_endpoint
-from ucp_base_operator import UcpBaseOperator
+except ImportError:
+    from shipyard_airflow.plugins.service_endpoint import ucp_service_endpoint
+
+try:
+    from service_token import shipyard_service_token
+except ImportError:
+    from shipyard_airflow.plugins.service_token import shipyard_service_token
+
+try:
+    from ucp_base_operator import UcpBaseOperator
+except ImportError:
+    from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator
 
 LOG = logging.getLogger(__name__)
 
@@ -35,6 +47,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
 
     @apply_defaults
     def __init__(self,
+                 deckhand_design_ref=None,
+                 deckhand_svc_type='deckhand',
                  promenade_svc_endpoint=None,
                  promenade_svc_type='kubernetesprovisioner',
                  redeploy_server=None,
@@ -42,6 +56,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
                  *args, **kwargs):
         """Initialization of PromenadeBaseOperator object.
 
+        :param deckhand_design_ref: A URI reference to the design documents
+        :param deckhand_svc_type: Deckhand Service Type
         :param promenade_svc_endpoint: Promenade Service Endpoint
         :param promenade_svc_type: Promenade Service Type
         :param redeploy_server: Server to be redeployed
@@ -55,6 +71,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
                   pod_selector_pattern=[{'pod_pattern': 'promenade-api',
                                          'container': 'promenade-api'}],
                   *args, **kwargs)
+        self.deckhand_design_ref = deckhand_design_ref
+        self.deckhand_svc_type = deckhand_svc_type
         self.promenade_svc_endpoint = promenade_svc_endpoint
         self.promenade_svc_type = promenade_svc_type
         self.redeploy_server = redeploy_server
@@ -85,6 +103,26 @@ class PromenadeBaseOperator(UcpBaseOperator):
 
         LOG.info("Promenade endpoint is %s", self.promenade_svc_endpoint)
 
+        # Retrieve Deckhand Endpoint Information
+        deckhand_svc_endpoint = ucp_service_endpoint(
+            self, svc_type=self.deckhand_svc_type)
+
+        LOG.info("Deckhand endpoint is %s", deckhand_svc_endpoint)
+
+        # Form Deckhand Design Reference Path
+        # This URL will be used to retrieve the Site Design YAMLs
+        deckhand_path = "deckhand+" + deckhand_svc_endpoint
+        self.deckhand_design_ref = os.path.join(deckhand_path,
+                                                "revisions",
+                                                str(self.revision_id),
+                                                "rendered-documents")
+        if self.deckhand_design_ref:
+            LOG.info("Design YAMLs will be retrieved from %s",
+                     self.deckhand_design_ref)
+        else:
+            raise AirflowException("Unable to Retrieve Deckhand Revision "
+                                   "%d!" % self.revision_id)
+
 
 class PromenadeBaseOperatorPlugin(AirflowPlugin):
 

src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py

@@ -0,0 +1,105 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import logging
+import os
+import requests
+
+from airflow.exceptions import AirflowException
+from airflow.plugins_manager import AirflowPlugin
+
+try:
+    from promenade_base_operator import PromenadeBaseOperator
+except ImportError:
+    from shipyard_airflow.plugins.promenade_base_operator import (
+        PromenadeBaseOperator
+    )
+
+LOG = logging.getLogger(__name__)
+
+
+class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator):
+    """
+    Promenade Validate Site Design Operator
+
+    This operator will trigger promenade to invoke the validateDesign
+    Promenade API
+    """
+
+    def do_execute(self):
+        LOG.info("Validating site design...")
+
+        # Form Validation Endpoint
+        validation_endpoint = os.path.join(self.promenade_svc_endpoint,
+                                           'validatedesign')
+
+        LOG.info("Validation Endpoint is %s", validation_endpoint)
+
+        # Define Headers and Payload
+        headers = {
+            'Content-Type': 'application/json',
+            'X-Auth-Token': self.svc_token
+        }
+
+        payload = {
+            'rel': "design",
+            'href': self.deckhand_design_ref,
+            'type': "application/x-yaml"
+        }
+
+        # Requests Promenade to validate site design
+        LOG.info("Waiting for Promenade to validate site design...")
+
+        try:
+            design_validate_response = requests.post(validation_endpoint,
+                                                     headers=headers,
+                                                     data=json.dumps(payload))
+
+        except requests.exceptions.RequestException as e:
+            raise AirflowException(e)
+
+        # Convert response to string
+        validate_site_design = design_validate_response.text
+
+        # Print response
+        LOG.info("Retrieving Promenade validate site design response...")
+
+        try:
+            validate_site_design_dict = json.loads(validate_site_design)
+            LOG.info(validate_site_design_dict)
+
+        except json.JSONDecodeError as e:
+            raise AirflowException(e)
+
+        # Check if site design is valid
+        status = validate_site_design_dict.get('status',
+                                               'unspecified')
+
+        if status.lower() == 'success':
+            LOG.info("Promenade Site Design has been successfully validated")
+
+        else:
+            # Dump logs from Promenade pods
+            self.get_k8s_logs()
+
+            raise AirflowException("Promenade Site Design Validation Failed "
+                                   "with status: {}!".format(status))
+
+
+class PromenadeValidateSiteDesignOperatorPlugin(AirflowPlugin):
+
+    """Creates PromenadeValidateSiteDesginOperator in Airflow."""
+
+    name = 'promenade_validate_site_design_operator'
+    operators = [PromenadeValidateSiteDesignOperator]

src/bin/shipyard_airflow/shipyard_airflow/plugins/service_token.py

@@ -18,7 +18,10 @@ import time
 
 from airflow.exceptions import AirflowException
 
-from service_session import ucp_keystone_session
+try:
+    from service_session import ucp_keystone_session
+except ImportError:
+    from shipyard_airflow.plugins.service_session import ucp_keystone_session
 
 
 def shipyard_service_token(func):

src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py

@@ -20,9 +20,20 @@ from airflow.models import BaseOperator
 from airflow.plugins_manager import AirflowPlugin
 from airflow.utils.decorators import apply_defaults
 
-from get_k8s_logs import get_pod_logs
+try:
-from get_k8s_logs import K8sLoggingException
+    from get_k8s_logs import get_pod_logs
-from xcom_puller import XcomPuller
+except ImportError:
+    from shipyard_airflow.plugins.get_k8s_logs import get_pod_logs
+
+try:
+    from get_k8s_logs import K8sLoggingException
+except ImportError:
+    from shipyard_airflow.plugins.get_k8s_logs import K8sLoggingException
+
+try:
+    from xcom_puller import XcomPuller
+except ImportError:
+    from shipyard_airflow.plugins.xcom_puller import XcomPuller
 
 LOG = logging.getLogger(__name__)
 

src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py

@@ -20,9 +20,20 @@ from airflow.models import BaseOperator
 from airflow.plugins_manager import AirflowPlugin
 from airflow.utils.decorators import apply_defaults
 
-from service_endpoint import ucp_service_endpoint
+try:
-from xcom_puller import XcomPuller
+    from service_endpoint import ucp_service_endpoint
-from xcom_pusher import XcomPusher
+except ImportError:
+    from shipyard_airflow.plugins.service_endpoint import ucp_service_endpoint
+
+try:
+    from xcom_puller import XcomPuller
+except ImportError:
+    from shipyard_airflow.plugins.xcom_puller import XcomPuller
+
+try:
+    from xcom_pusher import XcomPusher
+except ImportError:
+    from shipyard_airflow.plugins.xcom_pusher import XcomPusher
 
 LOG = logging.getLogger(__name__)
 

src/bin/shipyard_airflow/tests/unit/control/test.conf

@@ -29,6 +29,10 @@ project_domain_name = default
 project_name = service
 user_domain_name = default
 username = shipyard
+[k8s_logs]
+ucp_namespace = ucp
+[promenade]
+service_type = kubernetesprovisioner
 [requests_config]
 airflow_log_connect_timeout = 5
 airflow_log_read_timeout = 300

tools/resources/shipyard.conf

@@ -42,6 +42,8 @@ user_domain_name = default
 username = shipyard
 [k8s_logs]
 ucp_namespace = ucp
+[promenade]
+service_type = kubernetesprovisioner
 [requests_config]
 airflow_log_connect_timeout = 5
 airflow_log_read_timeout = 300