airship
/
shipyard
Code Issues Proposed changes

Update deploy_site Dag for Integration Testing 

This is an ongoing effort to integrate Shipyard/Airflow
in CI/CD as patch sets across UCP are reveiwed and merged

The Armada client/API patch sets were reviewed and merged.

Changes made this time to the 'deploy_site' dag

1) Revert 'armada_deploy_site.wip' back to 'armada_deploy_site.py'
2) Add Concurrency_Check back into 'deploy_site' dag
3) Add Aramda Operator back into 'deploy_site' dag

Change-Id: Id2c58a908275e45fe694b5adb83eb325911a4850
This commit is contained in:
Anthony Lin 2017-11-02 21:50:24 +00:00
parent 986ae72349
commit a5be242c31
2 changed files with 23 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
shipyard_airflow/dags/armada_deploy_site.wip → shipyard_airflow/dags/armada_deploy_site.py
View File

28
shipyard_airflow/dags/deploy_site.py
View File

@ -16,9 +16,11 @@ from datetime import timedelta
import airflow
import failure_handlers
from airflow import DAG
from airflow.operators import ConcurrencyCheckOperator
from airflow.operators.python_operator import PythonOperator
from airflow.operators.subdag_operator import SubDagOperator
from armada_deploy_site import deploy_site_armada
from drydock_deploy_site import deploy_site_drydock
"""
NOTE: We are currently in the process of reviewing and merging patch sets
@ -27,13 +29,15 @@ work properly. In order to proceed with integration testing with the CI/CD
team, there is a need to rename the 'deploy_site.py' dag as 'deploy_site.wip'
while we sort out, review and merge the outstanding patch sets.
NOTE: We will only include the DryDock workflow here for our current testing.
Updates will be made to this test dag as we progress along with the integration
testing and UCP code reviews/merge.
NOTE: We will only include Concurrency_Check, Armada and DryDock workflow here
for our current testing. Updates will be made to this test dag as we progress
along with the integration testing and UCP code reviews/merge.
"""
PARENT_DAG_NAME = 'deploy_site'
ARMADA_BUILD_DAG_NAME = 'armada_build'
DAG_CONCURRENCY_CHECK_DAG_NAME = 'dag_concurrency_check'
DRYDOCK_BUILD_DAG_NAME = 'drydock_build'
PARENT_DAG_NAME = 'deploy_site'
default_args = {
'owner': 'airflow',
@ -64,6 +68,11 @@ def xcom_push(**kwargs):
action_xcom = PythonOperator(
task_id='action_xcom', dag=dag, python_callable=xcom_push)
concurrency_check = ConcurrencyCheckOperator(
task_id=DAG_CONCURRENCY_CHECK_DAG_NAME,
on_failure_callback=failure_handlers.step_failure_handler,
dag=dag)
drydock_build = SubDagOperator(
subdag=deploy_site_drydock(
PARENT_DAG_NAME, DRYDOCK_BUILD_DAG_NAME, args=default_args),
@ -71,5 +80,14 @@ drydock_build = SubDagOperator(
on_failure_callback=failure_handlers.step_failure_handler,
dag=dag)
armada_build = SubDagOperator(
subdag=deploy_site_armada(
PARENT_DAG_NAME, ARMADA_BUILD_DAG_NAME, args=default_args),
task_id=ARMADA_BUILD_DAG_NAME,
on_failure_callback=failure_handlers.step_failure_handler,
dag=dag)
# DAG Wiring
drydock_build.set_upstream(action_xcom)
concurrency_check.set_upstream(action_xcom)
drydock_build.set_upstream(concurrency_check)
armada_build.set_upstream(drydock_build)