From c8c8934c77a807a06f45a72f765d44f04f8b14b7 Mon Sep 17 00:00:00 2001 From: Phil Sphicas Date: Wed, 1 Apr 2020 16:21:52 -0700 Subject: [PATCH] Armada DAG: increase retries from 3 to 5 The current Armada DAG allows for 3 retries, reduced from 10 here: https://opendev.org/airship/shipyard/commit/bef8eecac111f00f79f8672db7101baa23b0488e This is sometimes insufficient, especially in cases where chart updates of underlying Airship or Kubernetes components are upgraded and pods are restarted underneath the airflow workers. The chart installation may be successful, but an Armada retry may still be consumed. This change increases the number of retries to 5. This will allow Armada to progress further through the manifest if there is a disruption after a chart is successfully installed. The tradeoff is that Armada may try to repeatedly install a chart that keeps failing in the same way, delaying the ultimate failure of the deployment. Change-Id: I1fad7b1d95af061595680a76d24c6d323b365a67 --- .../shipyard_airflow/dags/armada_deploy_site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/armada_deploy_site.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/armada_deploy_site.py index 6c39a983..9c375fc9 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/armada_deploy_site.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/armada_deploy_site.py @@ -49,7 +49,7 @@ def deploy_site_armada(parent_dag_name, child_dag_name, args): task_id='armada_post_apply', shipyard_conf=config_path, main_dag_name=parent_dag_name, - retries=3, + retries=5, dag=dag) # Get Helm Releases