airship
/
shipyard
Code Issues Proposed changes

Update DeckHand Dag/Operator 

We are facing an issue with the Deckhand Client [0]
and will need to skip the 'shipyard_retrieve_rendered_doc'
for the time being while we wait for a resolution for [0].
Note that 'shipyard_retrieve_rendered_doc' is not really needed
for actual deployment to work and was introduced as part of
the Shipyard-270 story.

Also fix bugs with the deckhand operator [2] after [1] was merged.

[0] https://github.com/att-comdev/deckhand/issues/18
[1] https://review.gerrithub.io/#/c/395922/

[2]

Error 1:

INFO - Subtask:   File "/usr/local/airflow/plugins/deckhand_operator.py", line 132, in execute
INFO - Subtask:     revision_id=revision_id)
INFO - Subtask: TypeError: keystone_token_get() got an unexpected keyword argument 'revision_id'

Error 2:

INFO - Subtask:   File "/usr/local/airflow/plugins/deckhand_operator.py", line 219, in deckhand_validate_site
INFO - Subtask:     context['revision_id'])
INFO - Subtask: KeyError: 'revision_id'

Change-Id: I03ce29b24d1b3ff6148f2eb4970614b1389a58a6
This commit is contained in:
Anthony Lin 2018-01-31 03:44:42 +00:00
parent 81860b6eb7
commit 2abf00876b
2 changed files with 23 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
shipyard_airflow/dags/deckhand_get_design.py
View File

@ -38,15 +38,21 @@ def get_design_deckhand(parent_dag_name, child_dag_name, args):
sub_dag_name=child_dag_name,
dag=dag)
shipyard_retrieve_rendered_doc = DeckhandOperator(
task_id='shipyard_retrieve_rendered_doc',
shipyard_conf=config_path,
action='shipyard_retrieve_rendered_doc',
main_dag_name=parent_dag_name,
sub_dag_name=child_dag_name,
dag=dag)
# TODO: There is a bug in Deckhand client that is preventing
# 'shipyard_retrieve_rendered_doc' from working properly. We
# are commenting this part of the workflow while we resolve
# the issue with the Deckhand client. We will uncomment this
# block once the issue with Deckhand is resolved.
#
# shipyard_retrieve_rendered_doc = DeckhandOperator(
# task_id='shipyard_retrieve_rendered_doc',
# shipyard_conf=config_path,
# action='shipyard_retrieve_rendered_doc',
# main_dag_name=parent_dag_name,
# sub_dag_name=child_dag_name,
# dag=dag)
# Define dependencies
shipyard_retrieve_rendered_doc.set_upstream(deckhand_design)
# shipyard_retrieve_rendered_doc.set_upstream(deckhand_design)
return dag

13
shipyard_airflow/plugins/deckhand_operator.py
View File

@ -121,7 +121,7 @@ class DeckhandOperator(BaseOperator):
if self.action == 'shipyard_retrieve_rendered_doc':
if revision_id:
self.retrieve_rendered_doc(context,
revision_id=revision_id)
revision_id)
else:
raise AirflowException('Invalid revision ID!')
@ -129,7 +129,7 @@ class DeckhandOperator(BaseOperator):
elif self.action == 'deckhand_validate_site_design':
if revision_id:
self.deckhand_validate_site(context,
revision_id=revision_id)
revision_id)
else:
raise AirflowException('Invalid revision ID!')
@ -216,7 +216,7 @@ class DeckhandOperator(BaseOperator):
if validation_status:
logging.info("Revision %d has been successfully validated",
context['revision_id'])
revision_id)
else:
raise AirflowException("DeckHand Site Design Validation Failed!")
@ -244,7 +244,12 @@ class DeckhandOperator(BaseOperator):
logging.info("Setting up DeckHand Client...")
# Set up DeckHand Client
deckhandclient = deckhand_client.Client(session=sess)
# NOTE: The communication between the Airflow workers and Deckhand
# happens via the 'internal' endpoint and not the 'public' endpoint.
# Hence we will need to override the 'endpoint_type' from the default
# 'public' endpoint to 'internal' endpoint.
deckhandclient = deckhand_client.Client(session=sess,
endpoint_type='internal')
logging.info("Retrieving Rendered Document...")