From 70eb1cef10bddd153e5ab3b0330881c9d84f9159 Mon Sep 17 00:00:00 2001 From: Bryan Strassner Date: Tue, 3 Jul 2018 16:09:03 -0500 Subject: [PATCH] Add udpate_software action to Shipyard Provides an action that can be invoked by a user to deploy only the software portion of a design. Change-Id: I880bdc245064364dfdd6a482a3cf2d2a293f6c0d --- docs/source/API-action-commands.rst | 16 +++ docs/source/API.rst | 8 +- docs/source/CLI.rst | 6 +- .../document_validators/document_validator.py | 2 +- .../control/action/action_validators.py | 51 +++++++-- .../control/action/actions_api.py | 8 +- .../control/helpers/configdocs_helper.py | 7 +- .../validate_deployment_configuration.py | 26 ++++- .../validate_deployment_strategy.py | 7 +- .../dags/common_step_factory.py | 4 +- .../shipyard_airflow/dags/update_software.py | 74 +++++++++++++ .../dags/validate_site_design.py | 58 ++++++---- .../plugins/concurrency_check_operator.py | 3 +- .../deckhand_create_site_action_tag.py | 6 +- .../plugins/ucp_preflight_check_operator.py | 2 + .../unit/control/test_action_validators.py | 104 +++++++++++++++--- .../tests/unit/control/test_actions_api.py | 8 +- .../shipyard_client/cli/get/commands.py | 2 +- .../shipyard_client/cli/help/output.py | 5 +- .../shipyard_client/cli/input_checks.py | 6 +- .../tests/unit/cli/test_input_checks.py | 1 + tools/execute_shipyard_action.sh | 4 +- tools/set_env | 14 +-- tools/shipyard.sh | 58 ++++++++++ tools/update_software.sh | 22 ++++ 25 files changed, 414 insertions(+), 88 deletions(-) create mode 100644 src/bin/shipyard_airflow/shipyard_airflow/dags/update_software.py create mode 100755 tools/shipyard.sh create mode 100755 tools/update_software.sh diff --git a/docs/source/API-action-commands.rst b/docs/source/API-action-commands.rst index 1f07e51c..b2efb081 100644 --- a/docs/source/API-action-commands.rst +++ b/docs/source/API-action-commands.rst @@ -54,6 +54,22 @@ update_site Applies a new committed configuration to the environment. The steps of update_site mirror those of :ref:`deploy_site`. +.. _update_software: + +update_software +~~~~~~~~~~~~~~~ +Triggers an update of the software in a site, using the latest committed +configuration documents. Steps, conceptually: + +#. Concurrency check + Prevents concurrent site modifications by conflicting + actions/workflows. +#. Validate design + Asks each involved Airship component to validate the design. This ensures + that the previously committed design is valid at the present time. +#. Armada build + Orchestrates Armada to configure software on the nodes as designed. + Actions under development ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/API.rst b/docs/source/API.rst index f52b9c93..53c9dd55 100644 --- a/docs/source/API.rst +++ b/docs/source/API.rst @@ -93,11 +93,11 @@ be several versions of documents in a site that are accessible via this API: default. (This behavior can be overridden by query parameters issued by the user of Shipyard) - The "Last Site Action" version represents the version of documents associated - with the last successful or failed site action. Site actions include 'deploy_site' - and 'update_site'. + with the last successful or failed site action. - The "Successful Site Action" version represents the version of documents - associated with the last successful site action. Site actions include 'deploy_site' - and 'update_site'. + associated with the last successful site action. +- Site actions include ``deploy_site``, ``update_site``, and + ``update_software``. All versions of documents rely upon Deckhand for storage. Shipyard uses the tagging features of Deckhand to find the appropriate Committed Documents, diff --git a/docs/source/CLI.rst b/docs/source/CLI.rst index 32bb2e80..8ac0cba8 100644 --- a/docs/source/CLI.rst +++ b/docs/source/CLI.rst @@ -582,9 +582,9 @@ get configdocs Retrieve documents loaded into Shipyard. The possible options include last committed, last site action, last successful site action and retrieval from -the Shipyard Buffer. Site actions include deploy_site and update_site. Note -that we can only select one of the options when we retrieve the documents -for a particular collection. +the Shipyard Buffer. Site actions include ``deploy_site``, ``update_site`` and +``update_software``. Note that only one option may be selected when retrieving +the documents for a particular collection. The command will compare the differences between the revisions specified if the collection option is not specified. Note that we can only compare between diff --git a/src/bin/shipyard_airflow/shipyard_airflow/common/document_validators/document_validator.py b/src/bin/shipyard_airflow/shipyard_airflow/common/document_validators/document_validator.py index 4700133b..329a33a7 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/common/document_validators/document_validator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/common/document_validators/document_validator.py @@ -132,7 +132,7 @@ class DocumentValidator(metaclass=abc.ABCMeta): self.doc_name, self.schema) # only proceed to validating the document if it is present. - LOG.debug("Generic document validaton complete. Proceeding to " + LOG.debug("Generic document validation complete. Proceeding to " "specific validation") self.do_validate() except DocumentLookupError as dle: diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/action/action_validators.py b/src/bin/shipyard_airflow/shipyard_airflow/control/action/action_validators.py index 1e3f7aac..ff8dcc08 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/action/action_validators.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/action/action_validators.py @@ -24,13 +24,15 @@ from shipyard_airflow.common.document_validators.document_validator_manager \ import DocumentValidationManager from shipyard_airflow.control import service_clients from shipyard_airflow.control.validators.validate_deployment_configuration \ - import ValidateDeploymentConfiguration + import ValidateDeploymentConfigurationBasic +from shipyard_airflow.control.validators.validate_deployment_configuration \ + import ValidateDeploymentConfigurationFull from shipyard_airflow.errors import ApiError LOG = logging.getLogger(__name__) -def validate_site_action(action): +def validate_site_action_full(action): """Validates that the deployment configuration is correctly set up Checks: @@ -46,24 +48,53 @@ def validate_site_action(action): """ validator = _SiteActionValidator( dh_client=service_clients.deckhand_client(), - action=action + action=action, + full_validation=True + ) + validator.validate() + + +def validate_site_action_basic(action): + """Validates that the DeploymentConfiguration is present + + Checks: + + - The deployment configuration from Deckhand using the design version + + - If the deployment configuration is missing, error + """ + validator = _SiteActionValidator( + dh_client=service_clients.deckhand_client(), + action=action, + full_validation=False ) validator.validate() class _SiteActionValidator: - """The validator object setup and used by the validate_site_action function + """The validator object used by the validate_site_action_ functions """ - def __init__(self, dh_client, action): + def __init__(self, dh_client, action, full_validation=True): self.action = action self.doc_revision = self._get_doc_revision() self.cont_on_fail = str(self._action_param( 'continue-on-fail')).lower() == 'true' - self.doc_val_mgr = DocumentValidationManager( - dh_client, - self.doc_revision, - [(ValidateDeploymentConfiguration, 'deployment-configuration')] - ) + if full_validation: + # Perform a complete validation + self.doc_val_mgr = DocumentValidationManager( + dh_client, + self.doc_revision, + [(ValidateDeploymentConfigurationFull, + 'deployment-configuration')] + ) + else: + # Perform a basic validation only + self.doc_val_mgr = DocumentValidationManager( + dh_client, + self.doc_revision, + [(ValidateDeploymentConfigurationBasic, + 'deployment-configuration')] + ) def validate(self): results = self.doc_val_mgr.validate() diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py index c2808f2a..76db168e 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py @@ -44,11 +44,15 @@ def _action_mappings(): return { 'deploy_site': { 'dag': 'deploy_site', - 'validators': [action_validators.validate_site_action] + 'validators': [action_validators.validate_site_action_full] }, 'update_site': { 'dag': 'update_site', - 'validators': [action_validators.validate_site_action] + 'validators': [action_validators.validate_site_action_full] + }, + 'update_software': { + 'dag': 'update_software', + 'validators': [action_validators.validate_site_action_basic] }, 'redeploy_server': { 'dag': 'redeploy_server', diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py index a48a54a6..dd63e65a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py @@ -35,7 +35,7 @@ from shipyard_airflow.control.helpers.deckhand_client import ( from shipyard_airflow.control.service_endpoints import ( Endpoints, get_endpoint, get_token) from shipyard_airflow.control.validators.validate_deployment_configuration \ - import ValidateDeploymentConfiguration + import ValidateDeploymentConfigurationFull from shipyard_airflow.errors import ApiError, AppError CONF = cfg.CONF @@ -497,12 +497,13 @@ class ConfigdocsHelper(object): return _format_validations_to_status(resp_msgs, error_count) def _get_shipyard_validations(self, revision_id): - # Run Shipyard's own validations + # Run Shipyard's own validations. try: sy_val_mgr = DocumentValidationManager( service_clients.deckhand_client(), revision_id, - [(ValidateDeploymentConfiguration, 'deployment-configuration')] + [(ValidateDeploymentConfigurationFull, + 'deployment-configuration')] ) return sy_val_mgr.validate() except Exception as ex: diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py b/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py index 7ab286ef..5246e0be 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_configuration.py @@ -26,14 +26,29 @@ from .validate_deployment_strategy import ValidateDeploymentStrategy LOG = logging.getLogger(__name__) -class ValidateDeploymentConfiguration(DocumentValidator): - """Validates the DeploymentConfiguration.""" +class ValidateDeploymentConfigurationBasic(DocumentValidator): + """Validates that the DeploymentConfiguration is present + + The base DocumentValidator ensures the document is present. + The Schema validation done separately ensures that the Armada Manifest + document is specified. + """ def __init__(self, **kwargs): super().__init__(**kwargs) schema = "shipyard/DeploymentConfiguration/v1" missing_severity = "Error" + def do_validate(self): + self.error_status = False + + +class ValidateDeploymentConfigurationFull( + ValidateDeploymentConfigurationBasic): + """Validates the DeploymentConfiguration + + Includes a triggered check for DeploymentStrategy + """ def do_validate(self): try: dep_strat_nm = ( @@ -42,7 +57,10 @@ class ValidateDeploymentConfiguration(DocumentValidator): self.add_triggered_validation(ValidateDeploymentStrategy, dep_strat_nm) - except KeyError: + except (KeyError, TypeError): + # need to check both KeyError for missing 'deployment_strategy' + # and TypeError for not subscriptable exception when + # 'physical_provisioner' is None self.val_msg_list.append(self.val_msg( name="DeploymentStrategyNotSpecified", error=False, @@ -55,4 +73,4 @@ class ValidateDeploymentConfiguration(DocumentValidator): "'all-at-once' is assumed, and deployment strategy will " "not be further validated") - self.error_status = False + super().do_validate() diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_strategy.py b/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_strategy.py index a55d4991..c7f3154a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_strategy.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_strategy.py @@ -46,6 +46,11 @@ def _get_node_lookup(revision_id): ).lookup +def _get_deployment_group_manager(groups, revision_id): + """Retrieves the deployment group manager""" + return DeploymentGroupManager(groups, _get_node_lookup(revision_id)) + + class ValidateDeploymentStrategy(DocumentValidator): """Validates the deployment strategy""" def __init__(self, **kwargs): @@ -57,7 +62,7 @@ class ValidateDeploymentStrategy(DocumentValidator): def do_validate(self): groups = self.doc_dict['groups'] try: - DeploymentGroupManager(groups, _get_node_lookup(self.revision)) + _get_deployment_group_manager(groups, self.revision) except DeploymentGroupCycleError as dgce: self.val_msg_list.append(self.val_msg( name=dgce.__class__.__name__, diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py index 61f4d2e6..fb155a42 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py @@ -105,6 +105,7 @@ class CommonStepFactory(object): dag=self.dag) def get_validate_site_design(self, + targets=None, task_id=dn.VALIDATE_SITE_DESIGN_DAG_NAME): """Generate the validate site design step @@ -115,7 +116,8 @@ class CommonStepFactory(object): subdag=validate_site_design( self.parent_dag_name, task_id, - args=self.default_args), + args=self.default_args, + targets=targets), task_id=task_id, on_failure_callback=step_failure_handler, dag=self.dag) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/update_software.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/update_software.py new file mode 100644 index 00000000..4d2bd7ea --- /dev/null +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/update_software.py @@ -0,0 +1,74 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from datetime import timedelta + +import airflow +from airflow import DAG + +from common_step_factory import CommonStepFactory +from validate_site_design import SOFTWARE + +"""update_software + +The top-level orchestration DAG for updating only the software components +using the Undercloud platform. +""" +PARENT_DAG_NAME = 'update_software' + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': airflow.utils.dates.days_ago(1), + 'email': [''], + 'email_on_failure': False, + 'email_on_retry': False, + 'provide_context': True, + 'retries': 0, + 'retry_delay': timedelta(seconds=30), +} + +dag = DAG(PARENT_DAG_NAME, default_args=default_args, schedule_interval=None) + +step_factory = CommonStepFactory(parent_dag_name=PARENT_DAG_NAME, + dag=dag, + default_args=default_args) + +action_xcom = step_factory.get_action_xcom() +concurrency_check = step_factory.get_concurrency_check() +deployment_configuration = step_factory.get_deployment_configuration() +validate_site_design = step_factory.get_validate_site_design( + targets=[SOFTWARE] +) +armada_build = step_factory.get_armada_build() +decide_airflow_upgrade = step_factory.get_decide_airflow_upgrade() +upgrade_airflow = step_factory.get_upgrade_airflow() +skip_upgrade_airflow = step_factory.get_skip_upgrade_airflow() +create_action_tag = step_factory.get_create_action_tag() + +# DAG Wiring +deployment_configuration.set_upstream(action_xcom) +validate_site_design.set_upstream([ + concurrency_check, + deployment_configuration +]) +armada_build.set_upstream(validate_site_design) +decide_airflow_upgrade.set_upstream(armada_build) +decide_airflow_upgrade.set_downstream([ + upgrade_airflow, + skip_upgrade_airflow +]) +create_action_tag.set_upstream([ + upgrade_airflow, + skip_upgrade_airflow +]) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py index ccbedb5a..9dbdaf4d 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py @@ -21,7 +21,11 @@ from airflow.operators import PromenadeValidateSiteDesignOperator from config_path import config_path -def validate_site_design(parent_dag_name, child_dag_name, args): +BAREMETAL = 'baremetal' +SOFTWARE = 'software' + + +def validate_site_design(parent_dag_name, child_dag_name, args, targets=None): """Subdag to delegate design verification to the UCP components There is no wiring of steps - they all execute in parallel @@ -30,32 +34,44 @@ def validate_site_design(parent_dag_name, child_dag_name, args): '{}.{}'.format(parent_dag_name, child_dag_name), default_args=args) - deckhand_validate_docs = DeckhandValidateSiteDesignOperator( + if targets is None: + targets = [BAREMETAL, SOFTWARE] + + # Always add Deckhand validations + DeckhandValidateSiteDesignOperator( task_id='deckhand_validate_site_design', shipyard_conf=config_path, main_dag_name=parent_dag_name, retries=1, - dag=dag) + dag=dag + ) - drydock_validate_docs = DrydockValidateDesignOperator( - task_id='drydock_validate_site_design', - shipyard_conf=config_path, - main_dag_name=parent_dag_name, - retries=1, - dag=dag) + if BAREMETAL in targets: + # Add Drydock and Promenade validations + DrydockValidateDesignOperator( + task_id='drydock_validate_site_design', + shipyard_conf=config_path, + main_dag_name=parent_dag_name, + retries=1, + dag=dag + ) - armada_validate_docs = ArmadaValidateDesignOperator( - task_id='armada_validate_site_design', - shipyard_conf=config_path, - main_dag_name=parent_dag_name, - retries=1, - dag=dag) + PromenadeValidateSiteDesignOperator( + task_id='promenade_validate_site_design', + shipyard_conf=config_path, + main_dag_name=parent_dag_name, + retries=1, + dag=dag + ) - promenade_validate_docs = PromenadeValidateSiteDesignOperator( - task_id='promenade_validate_site_design', - shipyard_conf=config_path, - main_dag_name=parent_dag_name, - retries=1, - dag=dag) + if SOFTWARE in targets: + # Add Armada validations + ArmadaValidateDesignOperator( + task_id='armada_validate_site_design', + shipyard_conf=config_path, + main_dag_name=parent_dag_name, + retries=1, + dag=dag + ) return dag diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/concurrency_check_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/concurrency_check_operator.py index 9e99eb59..56df07ef 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/concurrency_check_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/concurrency_check_operator.py @@ -28,7 +28,8 @@ DAG_RUN_SELECT_RUNNING_SQL = ("select dag_id, execution_date " AIRFLOW_DB = 'airflows_own_db' # each set in this list of sets indicates DAGs that shouldn't execute together -CONFLICTING_DAG_SETS = [set(['deploy_site', 'update_site', 'redeploy_server'])] +CONFLICTING_DAG_SETS = [set(['deploy_site', 'update_site', 'update_software', + 'redeploy_server'])] def find_conflicting_dag_set(dag_name, conflicting_dag_sets=None): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_create_site_action_tag.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_create_site_action_tag.py index e455f45c..15728a9a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_create_site_action_tag.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_create_site_action_tag.py @@ -111,7 +111,7 @@ class DeckhandCreateSiteActionTagOperator(DeckhandBaseOperator): task = ['armada_build'] task_result = {} - if self.main_dag_name == 'update_site': + if self.main_dag_name in ['update_site', 'update_software']: # NOTE: We will check the final state of the 'armada_build' task # as a 'success' means that all tasks preceding it would either # be in 'skipped' or 'success' state. A failure of 'armada_build' @@ -119,8 +119,8 @@ class DeckhandCreateSiteActionTagOperator(DeckhandBaseOperator): # to determine the success/failure of the 'deploy_site' workflow # with the final state of the 'armada_build' task. # - # NOTE: The 'update_site' workflow contains additional steps for - # upgrading of worker pods. + # NOTE: The 'update_site' and 'update_software' workflows contain + # additional steps for upgrading of worker pods. for k in ['skip_upgrade_airflow', 'upgrade_airflow']: task.append(k) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py index 256d091c..0393703b 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py @@ -108,6 +108,8 @@ class UcpHealthCheckOperator(BaseOperator): """ # If Drydock health check fails and continue-on-fail, continue # and create xcom key 'drydock_continue_on_fail' + # Note that 'update_software' does not interact with Drydock, and + # therefore does not use the continue-on-fail option. if (component == service_endpoint.DRYDOCK and self.action_info['parameters'].get( 'continue-on-fail', 'false').lower() == 'true' and diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py b/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py index 3196185d..bfe86bf8 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py @@ -18,8 +18,14 @@ import yaml import pytest +from shipyard_airflow.common.deployment_group.errors import ( + DeploymentGroupCycleError, + InvalidDeploymentGroupError, + InvalidDeploymentGroupNodeLookupError +) from shipyard_airflow.control.action.action_validators import ( - validate_site_action + validate_site_action_basic, + validate_site_action_full ) from shipyard_airflow.errors import ApiError from tests.unit.common.deployment_group.node_lookup_stubs import node_lookup @@ -70,10 +76,10 @@ class TestActionValidator: @mock.patch("shipyard_airflow.control.validators." "validate_deployment_strategy._get_node_lookup", return_value=node_lookup) - def test_validate_site_action(self, *args): + def test_validate_site_action_full(self, *args): """Test the function that runs the validator class""" try: - validate_site_action({ + validate_site_action_full({ 'id': '123', 'name': 'deploy_site', 'committed_rev_id': 1 @@ -87,12 +93,12 @@ class TestActionValidator: @mock.patch("shipyard_airflow.control.validators." "validate_deployment_strategy._get_node_lookup", return_value=node_lookup) - def test_validate_site_action_cycle(self, *args): + def test_validate_site_action_full_cycle(self, *args): """Test the function that runs the validator class with a deployment strategy that has a cycle in the groups """ with pytest.raises(ApiError) as apie: - validate_site_action({ + validate_site_action_full({ 'id': '123', 'name': 'deploy_site', 'committed_rev_id': 1 @@ -107,12 +113,12 @@ class TestActionValidator: @mock.patch("shipyard_airflow.control.validators." "validate_deployment_strategy._get_node_lookup", return_value=node_lookup) - def test_validate_site_action_missing_dep_strat(self, *args): + def test_validate_site_action_full_missing_dep_strat(self, *args): """Test the function that runs the validator class with a missing deployment strategy - specified, but not present """ with pytest.raises(ApiError) as apie: - validate_site_action({ + validate_site_action_full({ 'id': '123', 'name': 'deploy_site', 'committed_rev_id': 1 @@ -121,21 +127,21 @@ class TestActionValidator: assert apie.value.error_list[0]['name'] == 'DocumentNotFoundError' @mock.patch("shipyard_airflow.control.service_clients.deckhand_client", - return_value=fake_dh_doc_client('clean'), ds_name='defaulted') + return_value=fake_dh_doc_client('clean', ds_name='defaulted')) @mock.patch("shipyard_airflow.control.validators." "validate_deployment_strategy._get_node_lookup", return_value=node_lookup) - def test_validate_site_action_default_dep_strat(self, *args): + def test_validate_site_action_full_default_dep_strat(self, *args): """Test the function that runs the validator class with a defaulted deployment strategy (not specified) """ try: - validate_site_action({ + validate_site_action_full({ 'id': '123', 'name': 'deploy_site', 'committed_rev_id': 1 }) - except: + except Exception: # any exception is a failure assert False @@ -149,7 +155,7 @@ class TestActionValidator: deployment strategy that has a cycle in the groups """ with pytest.raises(ApiError) as apie: - validate_site_action({ + validate_site_action_full({ 'id': '123', 'name': 'deploy_site' }) @@ -160,17 +166,81 @@ class TestActionValidator: @mock.patch("shipyard_airflow.control.validators." "validate_deployment_strategy._get_node_lookup", return_value=node_lookup) - def test_validate_site_action_continue_failure(self, *args): - """Test the function that runs the validator class with a defaulted - deployment strategy (not specified) + def test_validate_site_action_full_continue_failure(self, *args): + """Test the function that runs the validator class with a missing + deployment strategy (not specified), but continue-on-fail specified """ try: - validate_site_action({ + validate_site_action_full({ 'id': '123', 'name': 'deploy_site', 'committed_rev_id': 1, 'parameters': {'continue-on-fail': 'true'} }) - except: + except Exception: # any exception is a failure assert False + + @mock.patch("shipyard_airflow.control.service_clients.deckhand_client", + return_value=fake_dh_doc_client('clean', ds_name='not-there')) + @mock.patch("shipyard_airflow.control.validators." + "validate_deployment_strategy._get_node_lookup", + return_value=node_lookup) + def test_validate_site_action_basic_missing_dep_strat(self, *args): + """Test the function that runs the validator class with a missing + deployment strategy - specified, but not present. This should be + ignored by the basic valdiator + """ + try: + validate_site_action_basic({ + 'id': '123', + 'name': 'deploy_site', + 'committed_rev_id': 1 + }) + except Exception: + # any exception is a failure + assert False + + @mock.patch("shipyard_airflow.control.service_clients.deckhand_client", + return_value=fake_dh_doc_client('clean')) + @mock.patch("shipyard_airflow.control.validators." + "validate_deployment_strategy._get_node_lookup", + return_value=node_lookup) + def test_validate_site_action_dep_strategy_exceptions(self, *args): + """Test the function that runs the validator class for exceptions""" + to_catch = [InvalidDeploymentGroupNodeLookupError, + InvalidDeploymentGroupError, DeploymentGroupCycleError] + for exc in to_catch: + with mock.patch( + "shipyard_airflow.control.validators." + "validate_deployment_strategy._get_deployment_group_manager", + side_effect=exc() + ): + with pytest.raises(ApiError) as apie: + validate_site_action_full({ + 'id': '123', + 'name': 'deploy_site', + 'committed_rev_id': 1 + }) + assert apie.value.description == 'InvalidConfigurationDocuments' + assert apie.value.error_list[0]['name'] == (exc.__name__) + + @mock.patch("shipyard_airflow.control.service_clients.deckhand_client", + return_value=fake_dh_doc_client('clean')) + @mock.patch("shipyard_airflow.control.validators." + "validate_deployment_strategy._get_node_lookup", + return_value=node_lookup) + @mock.patch("shipyard_airflow.control.validators." + "validate_deployment_strategy._get_deployment_group_manager", + side_effect=TypeError()) + def test_validate_site_action_dep_strategy_exception_other(self, *args): + """Test the function that runs the validator class""" + with pytest.raises(ApiError) as apie: + validate_site_action_full({ + 'id': '123', + 'name': 'deploy_site', + 'committed_rev_id': 1 + }) + assert apie.value.description == 'InvalidConfigurationDocuments' + assert apie.value.error_list[0]['name'] == ( + 'DocumentValidationProcessingError') diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py b/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py index 2db54d49..48694cd6 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py @@ -313,7 +313,7 @@ def test_create_action(): # with invalid input. fail. with mock.patch('shipyard_airflow.control.action.action_validators' - '.validate_site_action') as validator: + '.validate_site_action_full') as validator: try: action = action_resource.create_action( action={'name': 'broken', @@ -330,7 +330,7 @@ def test_create_action(): # with valid input and some parameters with mock.patch('shipyard_airflow.control.action.action_validators' - '.validate_site_action') as validator: + '.validate_site_action_full') as validator: try: action = action_resource.create_action( action={'name': 'deploy_site', @@ -351,7 +351,7 @@ def test_create_action(): # with valid input and no parameters with mock.patch('shipyard_airflow.control.action.action_validators' - '.validate_site_action') as validator: + '.validate_site_action_full') as validator: try: action = action_resource.create_action( action={'name': 'deploy_site'}, @@ -382,7 +382,7 @@ def test_create_action_validator_error(): # with valid input and some parameters with mock.patch('shipyard_airflow.control.action.action_validators' - '.validate_site_action', + '.validate_site_action_full', side_effect=ApiError(title='bad')): with pytest.raises(ApiError) as apie: action = action_resource.create_action( diff --git a/src/bin/shipyard_client/shipyard_client/cli/get/commands.py b/src/bin/shipyard_client/shipyard_client/cli/get/commands.py index 4ec183ef..be2704ed 100644 --- a/src/bin/shipyard_client/shipyard_client/cli/get/commands.py +++ b/src/bin/shipyard_client/shipyard_client/cli/get/commands.py @@ -222,7 +222,7 @@ def get_version(ctx, buffer, committed, last_site_action, 'successful or failed site action\n' '--successful-site-action for the documents associated with the ' 'last successful site action\n' - 'Site actions include deploy_site and update_site.') + 'Site actions are deploy_site, update_site, and update_software') elif len(optional_site_parameters) == 1: return optional_site_parameters[0] diff --git a/src/bin/shipyard_client/shipyard_client/cli/help/output.py b/src/bin/shipyard_client/shipyard_client/cli/help/output.py index de19ac2d..55c2d87d 100644 --- a/src/bin/shipyard_client/shipyard_client/cli/help/output.py +++ b/src/bin/shipyard_client/shipyard_client/cli/help/output.py @@ -37,9 +37,12 @@ The workflow actions that may be invoked using Shipyard deploy_site: Triggers the initial deployment of a site using the latest committed configuration documents. -update_site: Triggers the initial deployment of a site, using the latest +update_site: Triggers the update to a deployment of a site, using the latest committed configuration documents. +update_software: Starts an update that only exercises the software portion of + the commited configuration documents. + redeploy_server: Using parameters to indicate which server(s), triggers a redeployment of servers to the last committed design and secrets. diff --git a/src/bin/shipyard_client/shipyard_client/cli/input_checks.py b/src/bin/shipyard_client/shipyard_client/cli/input_checks.py index bba7a701..0103bf92 100644 --- a/src/bin/shipyard_client/shipyard_client/cli/input_checks.py +++ b/src/bin/shipyard_client/shipyard_client/cli/input_checks.py @@ -18,9 +18,11 @@ from arrow.parser import ParserError def check_action_command(ctx, action_command): """Verifies the action command is valid""" - if action_command not in ['deploy_site', 'update_site', 'redeploy_server']: + valid_commands = ['deploy_site', 'update_site', 'update_software', + 'redeploy_server'] + if action_command not in valid_commands: ctx.fail('Invalid action command. The action commands available are ' - 'deploy_site, update_site, and redeploy_server.') + ' {}'.format(', '.join(valid_commands))) def check_control_action(ctx, action): diff --git a/src/bin/shipyard_client/tests/unit/cli/test_input_checks.py b/src/bin/shipyard_client/tests/unit/cli/test_input_checks.py index 290ee2fc..14d00c91 100644 --- a/src/bin/shipyard_client/tests/unit/cli/test_input_checks.py +++ b/src/bin/shipyard_client/tests/unit/cli/test_input_checks.py @@ -176,6 +176,7 @@ def test_check_action_commands(): ctx = Mock(side_effect=Exception("failed")) input_checks.check_action_command(ctx, 'deploy_site') input_checks.check_action_command(ctx, 'update_site') + input_checks.check_action_command(ctx, 'update_software') input_checks.check_action_command(ctx, 'redeploy_server') ctx.fail.assert_not_called() diff --git a/tools/execute_shipyard_action.sh b/tools/execute_shipyard_action.sh index f563c7b5..13c2085a 100755 --- a/tools/execute_shipyard_action.sh +++ b/tools/execute_shipyard_action.sh @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This is a common script that is used by the deploy_site, update_site -# and redeploy_server scripts +# This is a common script that is used by the deploy_site, update_site, +# update_software and redeploy_server scripts set -ex diff --git a/tools/set_env b/tools/set_env index bb14adfa..e7532a07 100755 --- a/tools/set_env +++ b/tools/set_env @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# NOTE: If user is executing deploy_site, update_site or redeploy_server -# workflow from outside the cluster, e.g. from a remote jump server, then -# he/she will need to ensure that the DNS server is able to resolve the -# FQDN of the Shipyard and Keystone public URL (both will be pointing to -# the IP of the Ingress Controller). If the DNS resolution is not available, -# the user will need to ensure that the /etc/hosts file is properly updated -# before setting up the environment variables and running the worflow. +# NOTE: If a user is executing deploy_site, update_site, update_software or +# redeploy_server workflow from outside the cluster, e.g. from a remote jump +# server, then the user will need to ensure that the DNS server is able to +# resolve the FQDN of the Shipyard and Keystone public URL (both will be +# pointing to the IP of the Ingress Controller). If the DNS resolution is not +# available, the user will need to ensure that the /etc/hosts file is properly +# updated before setting up the environment variables and running the worflow. # Define Variable namespace="${namespace:-ucp}" diff --git a/tools/shipyard.sh b/tools/shipyard.sh new file mode 100755 index 00000000..7b2e005f --- /dev/null +++ b/tools/shipyard.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# User can run the script as they would execute the Shipyard CLI. +# For instance, to run the 'shipyard get actions' command, user can execute +# the following command after setting up the required environment variables: +# +# $ ./tools/shipyard.sh get actions +# + +# NOTE: If user is executing the script from outside the cluster, e.g. from +# a remote jump server, then he/she will need to ensure that the DNS server +# is able to resolve the FQDN of the Shipyard and Keystone public URL (both +# will be pointing to the IP of the Ingress Controller). If the DNS resolution +# is not available, the user will need to ensure that the /etc/hosts file is +# properly updated before running the script. + +# Commands requiring files as input utilize the pwd mounted into the container +# as the /target directory, e.g.: +# +# $ ./tools/shipyard.sh create configdocs design --filename=/target/afile.yaml + +# Get the path of the directory where the script is located +# Source Base Docker Command +SHIPYARD_HOSTPATH=${SHIPYARD_HOSTPATH:-"/target"} +NAMESPACE="${NAMESPACE:-ucp}" +SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master}" + +# Define Base Docker Command +base_docker_command=$(cat << EndOfCommand +sudo docker run -t --rm --net=host +-e http_proxy=${HTTP_PROXY} +-e https_proxy=${HTTPS_PROXY} +-e OS_AUTH_URL=${OS_AUTH_URL:-http://keystone.${NAMESPACE}.svc.cluster.local:80/v3} +-e OS_USERNAME=${OS_USERNAME:-shipyard} +-e OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME:-default} +-e OS_PASSWORD=${OS_PASSWORD:-password} +-e OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME:-default} +-e OS_PROJECT_NAME=${OS_PROJECT_NAME:-service} +EndOfCommand +) + +# Execute Shipyard CLI +${base_docker_command} -v "$(pwd)":"${SHIPYARD_HOSTPATH}" "${SHIPYARD_IMAGE}" $@ diff --git a/tools/update_software.sh b/tools/update_software.sh new file mode 100755 index 00000000..6e83e3f7 --- /dev/null +++ b/tools/update_software.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Source environment variables +source set_env + +# Execute shipyard action for update_software +bash execute_shipyard_action.sh 'update_software'