From aca1b60f22df65341eedc42ebc72e95cd54416b7 Mon Sep 17 00:00:00 2001 From: One-Fine-Day Date: Fri, 16 Feb 2018 15:23:00 -0600 Subject: [PATCH] Promenade validateDesign for Shipyard SHIPYARD-342: Shipyard Integration with Promenade to Validate Design Calls Promenade validateDesign API to validate site design. Change-Id: Ia763983ed9857d4e5b13cfb11d3654e75e6578a4 --- charts/shipyard/values.yaml | 2 + docs/source/_static/shipyard.conf.sample | 12 ++ .../etc/shipyard/shipyard.conf.sample | 12 ++ .../shipyard_airflow/conf/config.py | 23 +++- .../control/helpers/configdocs_helper.py | 4 + .../control/service_endpoints.py | 2 + .../dags/validate_site_design.py | 9 ++ .../plugins/promenade_base_operator.py | 44 +++++++- .../plugins/promenade_validate_site_design.py | 105 ++++++++++++++++++ .../shipyard_airflow/plugins/service_token.py | 5 +- .../plugins/ucp_base_operator.py | 17 ++- .../plugins/ucp_preflight_check_operator.py | 17 ++- .../tests/unit/control/test.conf | 4 + tools/resources/shipyard.conf | 2 + 14 files changed, 244 insertions(+), 14 deletions(-) create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index c67f0da5..e99aacd7 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -373,6 +373,8 @@ conf: destroy_node_query_interval: 30 destroy_node_task_timeout: 900 cluster_join_check_backoff_time: 120 + promenade: + service_type: kubernetesprovisioner keystone_authtoken: delay_auth_decision: true auth_type: password diff --git a/docs/source/_static/shipyard.conf.sample b/docs/source/_static/shipyard.conf.sample index 616d0a94..7362bb69 100644 --- a/docs/source/_static/shipyard.conf.sample +++ b/docs/source/_static/shipyard.conf.sample @@ -294,6 +294,18 @@ #named_log_levels = keystoneauth:20,keystonemiddleware:20 +[promenade] + +# +# From shipyard_api +# + +# The service type for the service playing the role of Promenade. The specified +# type is used to perform the service lookup in the Keystone service catalog. +# (string value) +#service_type = kubernetesprovisioner + + [requests_config] # diff --git a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample index 616d0a94..7362bb69 100644 --- a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample +++ b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample @@ -294,6 +294,18 @@ #named_log_levels = keystoneauth:20,keystonemiddleware:20 +[promenade] + +# +# From shipyard_api +# + +# The service type for the service playing the role of Promenade. The specified +# type is used to perform the service lookup in the Keystone service catalog. +# (string value) +#service_type = kubernetesprovisioner + + [requests_config] # diff --git a/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py b/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py index 3575ca18..712cbe05 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py @@ -89,7 +89,7 @@ SECTIONS = [ ), ConfigSection( name='shipyard', - title='Shipyard connection info', + title='Shipyard Connection Info', options=[ cfg.StrOpt( 'service_type', @@ -104,7 +104,7 @@ SECTIONS = [ ), ConfigSection( name='deckhand', - title='Deckhand connection info', + title='Deckhand Connection Info', options=[ cfg.StrOpt( 'service_type', @@ -119,7 +119,7 @@ SECTIONS = [ ), ConfigSection( name='armada', - title='Armada connection info', + title='Armada Connection Info', options=[ cfg.StrOpt( 'service_type', @@ -134,7 +134,7 @@ SECTIONS = [ ), ConfigSection( name='drydock', - title='Drydock connection info', + title='Drydock Connection Info', options=[ cfg.StrOpt( 'service_type', @@ -147,6 +147,21 @@ SECTIONS = [ ), ] ), + ConfigSection( + name='promenade', + title='Promenade Connection Info', + options=[ + cfg.StrOpt( + 'service_type', + default='kubernetesprovisioner', + help=( + 'The service type for the service playing the role ' + 'of Promenade. The specified type is used to perform ' + 'the service lookup in the Keystone service catalog.' + ) + ), + ] + ), ConfigSection( name='requests_config', title='Requests Configuration', diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py index 7a07dd85..a48a54a6 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py @@ -732,6 +732,10 @@ def _get_validation_endpoints(): 'name': 'Armada', 'url': val_ep.format(get_endpoint(Endpoints.ARMADA)) }, + { + 'name': 'Promenade', + 'url': val_ep.format(get_endpoint(Endpoints.PROMENADE)) + } ] diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/service_endpoints.py b/src/bin/shipyard_airflow/shipyard_airflow/control/service_endpoints.py index bb2bd591..f1b17a04 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/service_endpoints.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/service_endpoints.py @@ -39,6 +39,7 @@ class Endpoints(enum.Enum): DRYDOCK = 'drydock' ARMADA = 'armada' DECKHAND = 'deckhand' + PROMENADE = 'promenade' def _get_service_type(endpoint): @@ -57,6 +58,7 @@ def _get_service_type(endpoint): Endpoints.DRYDOCK: CONF.drydock.service_type, Endpoints.ARMADA: CONF.armada.service_type, Endpoints.DECKHAND: CONF.deckhand.service_type, + Endpoints.PROMENADE: CONF.promenade.service_type } return endpoint_values.get(endpoint) raise AppError( diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py index 4cbc5f17..5925f43a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py @@ -16,6 +16,7 @@ from airflow.models import DAG from airflow.operators import ArmadaValidateDesignOperator from airflow.operators import DeckhandValidateSiteDesignOperator from airflow.operators import DrydockValidateDesignOperator +from airflow.operators import PromenadeValidateSiteDesignOperator from config_path import config_path @@ -52,4 +53,12 @@ def validate_site_design(parent_dag_name, child_dag_name, args): retries=3, dag=dag) + promenade_validate_docs = PromenadeValidateSiteDesignOperator( + task_id='promenade_validate_site_design', + shipyard_conf=config_path, + main_dag_name=parent_dag_name, + sub_dag_name=child_dag_name, + retries=3, + dag=dag) + return dag diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py index e4987acd..00e7d1f6 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py @@ -12,14 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import os from airflow.utils.decorators import apply_defaults from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from service_endpoint import ucp_service_endpoint -from service_token import shipyard_service_token -from ucp_base_operator import UcpBaseOperator +try: + from service_endpoint import ucp_service_endpoint +except ImportError: + from shipyard_airflow.plugins.service_endpoint import ucp_service_endpoint + +try: + from service_token import shipyard_service_token +except ImportError: + from shipyard_airflow.plugins.service_token import shipyard_service_token + +try: + from ucp_base_operator import UcpBaseOperator +except ImportError: + from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator LOG = logging.getLogger(__name__) @@ -35,6 +47,8 @@ class PromenadeBaseOperator(UcpBaseOperator): @apply_defaults def __init__(self, + deckhand_design_ref=None, + deckhand_svc_type='deckhand', promenade_svc_endpoint=None, promenade_svc_type='kubernetesprovisioner', redeploy_server=None, @@ -42,6 +56,8 @@ class PromenadeBaseOperator(UcpBaseOperator): *args, **kwargs): """Initialization of PromenadeBaseOperator object. + :param deckhand_design_ref: A URI reference to the design documents + :param deckhand_svc_type: Deckhand Service Type :param promenade_svc_endpoint: Promenade Service Endpoint :param promenade_svc_type: Promenade Service Type :param redeploy_server: Server to be redeployed @@ -55,6 +71,8 @@ class PromenadeBaseOperator(UcpBaseOperator): pod_selector_pattern=[{'pod_pattern': 'promenade-api', 'container': 'promenade-api'}], *args, **kwargs) + self.deckhand_design_ref = deckhand_design_ref + self.deckhand_svc_type = deckhand_svc_type self.promenade_svc_endpoint = promenade_svc_endpoint self.promenade_svc_type = promenade_svc_type self.redeploy_server = redeploy_server @@ -85,6 +103,26 @@ class PromenadeBaseOperator(UcpBaseOperator): LOG.info("Promenade endpoint is %s", self.promenade_svc_endpoint) + # Retrieve Deckhand Endpoint Information + deckhand_svc_endpoint = ucp_service_endpoint( + self, svc_type=self.deckhand_svc_type) + + LOG.info("Deckhand endpoint is %s", deckhand_svc_endpoint) + + # Form Deckhand Design Reference Path + # This URL will be used to retrieve the Site Design YAMLs + deckhand_path = "deckhand+" + deckhand_svc_endpoint + self.deckhand_design_ref = os.path.join(deckhand_path, + "revisions", + str(self.revision_id), + "rendered-documents") + if self.deckhand_design_ref: + LOG.info("Design YAMLs will be retrieved from %s", + self.deckhand_design_ref) + else: + raise AirflowException("Unable to Retrieve Deckhand Revision " + "%d!" % self.revision_id) + class PromenadeBaseOperatorPlugin(AirflowPlugin): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py new file mode 100644 index 00000000..dab852af --- /dev/null +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py @@ -0,0 +1,105 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging +import os +import requests + +from airflow.exceptions import AirflowException +from airflow.plugins_manager import AirflowPlugin + +try: + from promenade_base_operator import PromenadeBaseOperator +except ImportError: + from shipyard_airflow.plugins.promenade_base_operator import ( + PromenadeBaseOperator + ) + +LOG = logging.getLogger(__name__) + + +class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator): + """ + Promenade Validate Site Design Operator + + This operator will trigger promenade to invoke the validateDesign + Promenade API + """ + + def do_execute(self): + LOG.info("Validating site design...") + + # Form Validation Endpoint + validation_endpoint = os.path.join(self.promenade_svc_endpoint, + 'validatedesign') + + LOG.info("Validation Endpoint is %s", validation_endpoint) + + # Define Headers and Payload + headers = { + 'Content-Type': 'application/json', + 'X-Auth-Token': self.svc_token + } + + payload = { + 'rel': "design", + 'href': self.deckhand_design_ref, + 'type': "application/x-yaml" + } + + # Requests Promenade to validate site design + LOG.info("Waiting for Promenade to validate site design...") + + try: + design_validate_response = requests.post(validation_endpoint, + headers=headers, + data=json.dumps(payload)) + + except requests.exceptions.RequestException as e: + raise AirflowException(e) + + # Convert response to string + validate_site_design = design_validate_response.text + + # Print response + LOG.info("Retrieving Promenade validate site design response...") + + try: + validate_site_design_dict = json.loads(validate_site_design) + LOG.info(validate_site_design_dict) + + except json.JSONDecodeError as e: + raise AirflowException(e) + + # Check if site design is valid + status = validate_site_design_dict.get('status', + 'unspecified') + + if status.lower() == 'success': + LOG.info("Promenade Site Design has been successfully validated") + + else: + # Dump logs from Promenade pods + self.get_k8s_logs() + + raise AirflowException("Promenade Site Design Validation Failed " + "with status: {}!".format(status)) + + +class PromenadeValidateSiteDesignOperatorPlugin(AirflowPlugin): + + """Creates PromenadeValidateSiteDesginOperator in Airflow.""" + + name = 'promenade_validate_site_design_operator' + operators = [PromenadeValidateSiteDesignOperator] diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/service_token.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/service_token.py index 6191416b..7bdd61bf 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/service_token.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/service_token.py @@ -18,7 +18,10 @@ import time from airflow.exceptions import AirflowException -from service_session import ucp_keystone_session +try: + from service_session import ucp_keystone_session +except ImportError: + from shipyard_airflow.plugins.service_session import ucp_keystone_session def shipyard_service_token(func): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py index 96fae01b..bd2176b4 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py @@ -20,9 +20,20 @@ from airflow.models import BaseOperator from airflow.plugins_manager import AirflowPlugin from airflow.utils.decorators import apply_defaults -from get_k8s_logs import get_pod_logs -from get_k8s_logs import K8sLoggingException -from xcom_puller import XcomPuller +try: + from get_k8s_logs import get_pod_logs +except ImportError: + from shipyard_airflow.plugins.get_k8s_logs import get_pod_logs + +try: + from get_k8s_logs import K8sLoggingException +except ImportError: + from shipyard_airflow.plugins.get_k8s_logs import K8sLoggingException + +try: + from xcom_puller import XcomPuller +except ImportError: + from shipyard_airflow.plugins.xcom_puller import XcomPuller LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py index 04755c71..c6bb820b 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py @@ -20,9 +20,20 @@ from airflow.models import BaseOperator from airflow.plugins_manager import AirflowPlugin from airflow.utils.decorators import apply_defaults -from service_endpoint import ucp_service_endpoint -from xcom_puller import XcomPuller -from xcom_pusher import XcomPusher +try: + from service_endpoint import ucp_service_endpoint +except ImportError: + from shipyard_airflow.plugins.service_endpoint import ucp_service_endpoint + +try: + from xcom_puller import XcomPuller +except ImportError: + from shipyard_airflow.plugins.xcom_puller import XcomPuller + +try: + from xcom_pusher import XcomPusher +except ImportError: + from shipyard_airflow.plugins.xcom_pusher import XcomPusher LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/tests/unit/control/test.conf b/src/bin/shipyard_airflow/tests/unit/control/test.conf index 22ea897f..584b92ad 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test.conf +++ b/src/bin/shipyard_airflow/tests/unit/control/test.conf @@ -29,6 +29,10 @@ project_domain_name = default project_name = service user_domain_name = default username = shipyard +[k8s_logs] +ucp_namespace = ucp +[promenade] +service_type = kubernetesprovisioner [requests_config] airflow_log_connect_timeout = 5 airflow_log_read_timeout = 300 diff --git a/tools/resources/shipyard.conf b/tools/resources/shipyard.conf index e35d4e3c..7665e3a8 100644 --- a/tools/resources/shipyard.conf +++ b/tools/resources/shipyard.conf @@ -42,6 +42,8 @@ user_domain_name = default username = shipyard [k8s_logs] ucp_namespace = ucp +[promenade] +service_type = kubernetesprovisioner [requests_config] airflow_log_connect_timeout = 5 airflow_log_read_timeout = 300