airship
/
shipyard
Code Issues Proposed changes

Shipyard-270 - Rendered document schema validation check 

The story requires Shipyard to get the rendered document from
Deckhand before proceeding with the rest of the workflow.

This is to ensure that the schema validation at Deckhand is
good and to fail early if need be.

This p.s. attempts to make use of the deckhand client to get
the list of rendered documents. Note that we will need to install
the DeckHand client libraries in the Airflow image in order for
this to happen.

Change-Id: I22f4165be101edfa4cef9bead6dd3d3765146c6e
This commit is contained in:
Anthony Lin 2018-01-23 09:21:10 +00:00
parent 5cc0b5b986
commit 8d486b22db
3 changed files with 85 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
images/airflow/Dockerfile
View File

@ -66,8 +66,10 @@ RUN pip3 install -r /tmp/requirements.txt
# https://github.com/puckel/docker-airflow/issues/77
RUN pip3 uninstall -y snakebite || true
RUN pip3 install -e git://github.com/att-comdev/drydock.git#egg=drydock_provisioner
# Install Armada, DeckHand and DryDock Client Libraries
RUN pip3 install -e git://github.com/att-comdev/armada.git#egg=armada
RUN pip3 install -e git://github.com/att-comdev/deckhand.git#egg=deckhand
RUN pip3 install -e git://github.com/att-comdev/drydock.git#egg=drydock_provisioner
# Create airflow user
RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow

11
shipyard_airflow/dags/deckhand_get_design.py
View File

@ -38,4 +38,15 @@ def get_design_deckhand(parent_dag_name, child_dag_name, args):
sub_dag_name=child_dag_name,
dag=dag)
shipyard_retrieve_rendered_doc = DeckhandOperator(
task_id='shipyard_retrieve_rendered_doc',
shipyard_conf=config_path,
action='shipyard_retrieve_rendered_doc',
main_dag_name=parent_dag_name,
sub_dag_name=child_dag_name,
dag=dag)
# Define dependencies
shipyard_retrieve_rendered_doc.set_upstream(deckhand_design)
return dag

85
shipyard_airflow/plugins/deckhand_operator.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import configparser
import logging
import os
import requests
@ -21,7 +22,10 @@ from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from airflow.plugins_manager import AirflowPlugin
from airflow.exceptions import AirflowException
from keystoneauth1.identity import v3 as keystone_v3
from keystoneauth1 import session as keystone_session
from deckhand.client import deckhand_client
from service_endpoint import ucp_service_endpoint
from service_token import shipyard_service_token
@ -57,6 +61,7 @@ class DeckhandOperator(BaseOperator):
# Initialize Variables
deckhand_design_version = None
redeploy_server = None
revision_id = None
# Define task_instance
task_instance = context['task_instance']
@ -100,20 +105,33 @@ class DeckhandOperator(BaseOperator):
else:
raise AirflowException('Failed to retrieve revision ID!')
# Retrieve revision_id from xcom
# Note that in the case of 'deploy_site', the dag_id will
# be 'deploy_site.deckhand_get_design_version' for the
# 'deckhand_get_design_version' task. We need to extract
# the xcom value from it in order to get the value of the
# last committed revision ID
revision_id = task_instance.xcom_pull(
task_ids='deckhand_get_design_version',
dag_id=self.main_dag_name + '.deckhand_get_design_version')
logging.info("Revision ID is %d", revision_id)
# Retrieve Rendered Document from DeckHand
if self.action == 'shipyard_retrieve_rendered_doc':
if revision_id:
self.retrieve_rendered_doc(context,
revision_id=revision_id)
else:
raise AirflowException('Invalid revision ID!')
# Validate Design using DeckHand
elif self.action == 'deckhand_validate_site_design':
# Retrieve revision_id from xcom
# Note that in the case of 'deploy_site', the dag_id will
# be 'deploy_site.deckhand_get_design_version' for the
# 'deckhand_get_design_version' task. We need to extract
# the xcom value from it in order to get the value of the
# last committed revision ID
context['revision_id'] = task_instance.xcom_pull(
task_ids='deckhand_get_design_version',
dag_id=self.main_dag_name + '.deckhand_get_design_version')
logging.info("Revision ID is %d", context['revision_id'])
self.deckhand_validate_site(context)
if revision_id:
self.deckhand_validate_site(context,
revision_id=revision_id)
else:
raise AirflowException('Invalid revision ID!')
# No action to perform
else:
@ -160,14 +178,14 @@ class DeckhandOperator(BaseOperator):
raise AirflowException("Failed to retrieve committed revision!")
@shipyard_service_token
def deckhand_validate_site(self, context):
def deckhand_validate_site(self, context, revision_id):
# Retrieve Keystone Token and assign to X-Auth-Token Header
x_auth_token = {"X-Auth-Token": context['svc_token']}
# Form Validation Endpoint
validation_endpoint = os.path.join(context['svc_endpoint'],
'revisions',
str(context['revision_id']),
str(revision_id),
'validations')
logging.info(validation_endpoint)
@ -202,6 +220,45 @@ class DeckhandOperator(BaseOperator):
else:
raise AirflowException("DeckHand Site Design Validation Failed!")
def retrieve_rendered_doc(self, context, revision_id):
# Initialize Variables
auth = None
keystone_auth = {}
rendered_doc = []
sess = None
# Read and parse shiyard.conf
config = configparser.ConfigParser()
config.read(self.shipyard_conf)
# Construct Session Argument
for attr in ('auth_url', 'password', 'project_domain_name',
'project_name', 'username', 'user_domain_name'):
keystone_auth[attr] = config.get('keystone_authtoken', attr)
# Set up keystone session
auth = keystone_v3.Password(**keystone_auth)
sess = keystone_session.Session(auth=auth)
logging.info("Setting up DeckHand Client...")
# Set up DeckHand Client
deckhandclient = deckhand_client.Client(session=sess)
logging.info("Retrieving Rendered Document...")
# Retrieve Rendered Document
try:
rendered_doc = deckhandclient.revisions.documents(revision_id,
rendered=True)
logging.info("Successfully Retrieved Rendered Document")
logging.info(rendered_doc)
except:
raise AirflowException("Failed to Retrieve Rendered Document!")
class DeckhandOperatorPlugin(AirflowPlugin):
name = 'deckhand_operator_plugin'