From c1bd1203c77bb658a29a34b8801e8839a7c3a14d Mon Sep 17 00:00:00 2001 From: hosingh000 Date: Wed, 18 Jul 2018 11:44:36 -0500 Subject: [PATCH] Block site_update if there is no host in MaaS/Drydock Added the feature in airflow to verify that MaaS list of BM hosts is not empty for shipyard update_site action. If the MaaS Machine list is empty, and the continue-on-fail parameter is not set to true (the default value is false), it will fail the shipyard steps to parepare and re-deploy the missing nodes in MaaS through DD. Caveat: this US did not have the requirement to compare the list of nodes in MaaS with the expected site Design. It simply checks for empty node list, and decide based on that. Change-Id: I5ba4a107fe2ae43728e5941570b6c88a436d7b12 --- .../dags/common_step_factory.py | 6 +- .../dags/drydock_deploy_site.py | 19 +++++- .../shipyard_airflow/dags/update_site.py | 2 +- .../plugins/drydock_base_operator.py | 16 +++++ .../plugins/drydock_verify_nodes.py | 60 +++++++++++++++++++ 5 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_verify_nodes.py diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py index 40979f8a..d27c9cde 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py @@ -174,7 +174,8 @@ class CommonStepFactory(object): on_failure_callback=step_failure_handler, dag=self.dag) - def get_drydock_build(self, task_id=dn.DRYDOCK_BUILD_DAG_NAME): + def get_drydock_build(self, task_id=dn.DRYDOCK_BUILD_DAG_NAME, + verify_nodes_exist=False): """Generate the drydock build step Drydock build does the hardware provisioning. @@ -183,7 +184,8 @@ class CommonStepFactory(object): subdag=deploy_site_drydock( self.parent_dag_name, task_id, - args=self.default_args), + args=self.default_args, + verify_nodes_exist=verify_nodes_exist), task_id=task_id, on_failure_callback=step_failure_handler, dag=self.dag) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/drydock_deploy_site.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/drydock_deploy_site.py index f2f32eb1..d4cf16ca 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/drydock_deploy_site.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/drydock_deploy_site.py @@ -18,6 +18,7 @@ try: from airflow.operators import DrydockNodesOperator from airflow.operators import DrydockPrepareSiteOperator from airflow.operators import DrydockVerifySiteOperator + from airflow.operators import DrydockVerifyNodesExistOperator from config_path import config_path except ImportError: from shipyard_airflow.plugins.drydock_nodes import \ @@ -26,10 +27,13 @@ except ImportError: DrydockPrepareSiteOperator from shipyard_airflow.plugins.drydock_verify_site import \ DrydockVerifySiteOperator + from shipyard_airflow.plugins.drydock_verify_nodes import \ + DrydockVerifyNodesExistOperator from shipyard_airflow.dags.config_path import config_path -def deploy_site_drydock(parent_dag_name, child_dag_name, args): +def deploy_site_drydock(parent_dag_name, child_dag_name, args, + verify_nodes_exist=False): ''' DryDock Subdag ''' @@ -37,6 +41,13 @@ def deploy_site_drydock(parent_dag_name, child_dag_name, args): '{}.{}'.format(parent_dag_name, child_dag_name), default_args=args) + if verify_nodes_exist: + drydock_verify_nodes_exist = DrydockVerifyNodesExistOperator( + task_id='verify_nodes_exist', + shipyard_conf=config_path, + main_dag_name=parent_dag_name, + dag=dag) + drydock_verify_site = DrydockVerifySiteOperator( task_id='verify_site', shipyard_conf=config_path, @@ -57,6 +68,10 @@ def deploy_site_drydock(parent_dag_name, child_dag_name, args): # Define dependencies drydock_prepare_site.set_upstream(drydock_verify_site) - drydock_nodes.set_upstream(drydock_prepare_site) + if verify_nodes_exist: + drydock_verify_nodes_exist.set_upstream(drydock_prepare_site) + drydock_nodes.set_upstream(drydock_verify_nodes_exist) + else: + drydock_nodes.set_upstream(drydock_prepare_site) return dag diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/update_site.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/update_site.py index 7026fd35..892fbb20 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/update_site.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/update_site.py @@ -58,7 +58,7 @@ preflight = step_factory.get_preflight() get_rendered_doc = step_factory.get_get_rendered_doc() deployment_configuration = step_factory.get_deployment_configuration() validate_site_design = step_factory.get_validate_site_design() -drydock_build = step_factory.get_drydock_build() +drydock_build = step_factory.get_drydock_build(verify_nodes_exist=True) armada_build = step_factory.get_armada_build() decide_airflow_upgrade = step_factory.get_decide_airflow_upgrade() upgrade_airflow = step_factory.get_upgrade_airflow() diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py index cb338150..27e8ec89 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py @@ -368,6 +368,22 @@ class DrydockBaseOperator(UcpBaseOperator): "Unable to retrieve subtask info!" ) + def get_nodes(self): + """ + Get the list of all the build data record for all nodes(hostname) + in raw dictionary format. + + Raises DrydockClientUseFailureException if the client raises an + exception + See: + https://att-comdev-drydock.readthedocs.io/en/latest/API.html + """ + try: + return self.drydock_client.get_nodes() + except errors.ClientError as client_error: + LOG.error("Drydock client failed to get nodes from Drydock.") + raise DrydockClientUseFailureException(client_error) + def get_successes_for_task(self, task_id, extend_success=True): """Discover the successful nodes based on the current task id. diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_verify_nodes.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_verify_nodes.py new file mode 100644 index 00000000..212888be --- /dev/null +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_verify_nodes.py @@ -0,0 +1,60 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from airflow.exceptions import AirflowException +from airflow.plugins_manager import AirflowPlugin + +try: + from drydock_base_operator import DrydockBaseOperator +except ImportError: + from shipyard_airflow.plugins.drydock_base_operator import \ + DrydockBaseOperator + +LOG = logging.getLogger(__name__) + + +class DrydockVerifyNodesExistOperator(DrydockBaseOperator): + + """Drydock Verify nodes exist Operator + + This operator will trigger drydock to verify node for + site update + + """ + + def do_execute(self): + + LOG.info('verify_nodes_exit was invoked.') + node_list = self.get_nodes() + continue_on_fail = self.action_info['parameters'].get( + 'continue-on-fail', 'false') + LOG.debug('node list is : {}'.format(node_list)) + LOG.debug('continue on fail is: {}'.format(continue_on_fail)) + + if not node_list and str(continue_on_fail).lower() != 'true': + msg = 'No nodes were found in MaaS, ' \ + 'and continue_on_fail is {} ' \ + '-> Fail Drydock prepare and ' \ + 'deply nodes.'.format(continue_on_fail) + LOG.error(msg) + raise AirflowException(msg) + + +class DrydockVerifyNodesExistOperatorPlugin(AirflowPlugin): + + """Creates DrydockVerifyNodesExistOperatorPlugin in Airflow.""" + + name = 'drydock_verify_nodes_exist_operator' + operators = [DrydockVerifyNodesExistOperator]