diff --git a/images/airflow/Dockerfile b/images/airflow/Dockerfile index 9d569785..61b7f678 100644 --- a/images/airflow/Dockerfile +++ b/images/airflow/Dockerfile @@ -66,8 +66,10 @@ RUN pip3 install -r /tmp/requirements.txt # https://github.com/puckel/docker-airflow/issues/77 RUN pip3 uninstall -y snakebite || true -RUN pip3 install -e git://github.com/att-comdev/drydock.git#egg=drydock_provisioner +# Install Armada, DeckHand and DryDock Client Libraries RUN pip3 install -e git://github.com/att-comdev/armada.git#egg=armada +RUN pip3 install -e git://github.com/att-comdev/deckhand.git#egg=deckhand +RUN pip3 install -e git://github.com/att-comdev/drydock.git#egg=drydock_provisioner # Create airflow user RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow diff --git a/shipyard_airflow/dags/deckhand_get_design.py b/shipyard_airflow/dags/deckhand_get_design.py index 79bd0e52..d673471e 100644 --- a/shipyard_airflow/dags/deckhand_get_design.py +++ b/shipyard_airflow/dags/deckhand_get_design.py @@ -38,4 +38,15 @@ def get_design_deckhand(parent_dag_name, child_dag_name, args): sub_dag_name=child_dag_name, dag=dag) + shipyard_retrieve_rendered_doc = DeckhandOperator( + task_id='shipyard_retrieve_rendered_doc', + shipyard_conf=config_path, + action='shipyard_retrieve_rendered_doc', + main_dag_name=parent_dag_name, + sub_dag_name=child_dag_name, + dag=dag) + + # Define dependencies + shipyard_retrieve_rendered_doc.set_upstream(deckhand_design) + return dag diff --git a/shipyard_airflow/plugins/deckhand_operator.py b/shipyard_airflow/plugins/deckhand_operator.py index e262c8ba..206a3d6e 100644 --- a/shipyard_airflow/plugins/deckhand_operator.py +++ b/shipyard_airflow/plugins/deckhand_operator.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import configparser import logging import os import requests @@ -21,7 +22,10 @@ from airflow.models import BaseOperator from airflow.utils.decorators import apply_defaults from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException +from keystoneauth1.identity import v3 as keystone_v3 +from keystoneauth1 import session as keystone_session +from deckhand.client import deckhand_client from service_endpoint import ucp_service_endpoint from service_token import shipyard_service_token @@ -57,6 +61,7 @@ class DeckhandOperator(BaseOperator): # Initialize Variables deckhand_design_version = None redeploy_server = None + revision_id = None # Define task_instance task_instance = context['task_instance'] @@ -100,20 +105,33 @@ class DeckhandOperator(BaseOperator): else: raise AirflowException('Failed to retrieve revision ID!') + # Retrieve revision_id from xcom + # Note that in the case of 'deploy_site', the dag_id will + # be 'deploy_site.deckhand_get_design_version' for the + # 'deckhand_get_design_version' task. We need to extract + # the xcom value from it in order to get the value of the + # last committed revision ID + revision_id = task_instance.xcom_pull( + task_ids='deckhand_get_design_version', + dag_id=self.main_dag_name + '.deckhand_get_design_version') + + logging.info("Revision ID is %d", revision_id) + + # Retrieve Rendered Document from DeckHand + if self.action == 'shipyard_retrieve_rendered_doc': + if revision_id: + self.retrieve_rendered_doc(context, + revision_id=revision_id) + else: + raise AirflowException('Invalid revision ID!') + # Validate Design using DeckHand elif self.action == 'deckhand_validate_site_design': - # Retrieve revision_id from xcom - # Note that in the case of 'deploy_site', the dag_id will - # be 'deploy_site.deckhand_get_design_version' for the - # 'deckhand_get_design_version' task. We need to extract - # the xcom value from it in order to get the value of the - # last committed revision ID - context['revision_id'] = task_instance.xcom_pull( - task_ids='deckhand_get_design_version', - dag_id=self.main_dag_name + '.deckhand_get_design_version') - - logging.info("Revision ID is %d", context['revision_id']) - self.deckhand_validate_site(context) + if revision_id: + self.deckhand_validate_site(context, + revision_id=revision_id) + else: + raise AirflowException('Invalid revision ID!') # No action to perform else: @@ -160,14 +178,14 @@ class DeckhandOperator(BaseOperator): raise AirflowException("Failed to retrieve committed revision!") @shipyard_service_token - def deckhand_validate_site(self, context): + def deckhand_validate_site(self, context, revision_id): # Retrieve Keystone Token and assign to X-Auth-Token Header x_auth_token = {"X-Auth-Token": context['svc_token']} # Form Validation Endpoint validation_endpoint = os.path.join(context['svc_endpoint'], 'revisions', - str(context['revision_id']), + str(revision_id), 'validations') logging.info(validation_endpoint) @@ -202,6 +220,45 @@ class DeckhandOperator(BaseOperator): else: raise AirflowException("DeckHand Site Design Validation Failed!") + def retrieve_rendered_doc(self, context, revision_id): + + # Initialize Variables + auth = None + keystone_auth = {} + rendered_doc = [] + sess = None + + # Read and parse shiyard.conf + config = configparser.ConfigParser() + config.read(self.shipyard_conf) + + # Construct Session Argument + for attr in ('auth_url', 'password', 'project_domain_name', + 'project_name', 'username', 'user_domain_name'): + keystone_auth[attr] = config.get('keystone_authtoken', attr) + + # Set up keystone session + auth = keystone_v3.Password(**keystone_auth) + sess = keystone_session.Session(auth=auth) + + logging.info("Setting up DeckHand Client...") + + # Set up DeckHand Client + deckhandclient = deckhand_client.Client(session=sess) + + logging.info("Retrieving Rendered Document...") + + # Retrieve Rendered Document + try: + rendered_doc = deckhandclient.revisions.documents(revision_id, + rendered=True) + + logging.info("Successfully Retrieved Rendered Document") + logging.info(rendered_doc) + + except: + raise AirflowException("Failed to Retrieve Rendered Document!") + class DeckhandOperatorPlugin(AirflowPlugin): name = 'deckhand_operator_plugin'