From 01c62f563b08855a71714586c558aebb5ad9de6a Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Thu, 14 Dec 2017 09:06:10 -0600 Subject: [PATCH] Add design_ref to template context For Bootactions that use the template pipeline segment, add the `action.design_ref` field to the context for Jinja2. - Add design_ref to the render_assets method signature - Update calls to include the contextual design_ref - Add unit test for design_ref rendering Change-Id: I81017e050b6f1c0a3e66ee824ecb8ffd154e45dd --- docs/source/bootaction.rst | 6 ++ drydock_provisioner/control/bootaction.py | 1 + .../ingester/plugins/deckhand.py | 1 + drydock_provisioner/ingester/plugins/yaml.py | 1 + drydock_provisioner/objects/bootaction.py | 11 ++- .../orchestrator/orchestrator.py | 14 +++- drydock_provisioner/schemas/bootaction.yaml | 2 + .../postgres/test_bootaction_signalling.py | 41 +++++++++++ .../test_postgres_bootaction_status.py | 2 +- tests/unit/test_bootaction_asset_render.py | 23 +++++- tests/unit/test_bootaction_tarbuilder.py | 2 +- tests/unit/test_ingester_bootaction.py | 6 +- tests/yaml_samples/bootaction.yaml | 31 -------- tests/yaml_samples/deckhand_bootaction.yaml | 53 ++++++++++++++ tests/yaml_samples/deckhand_fullsite.yaml | 70 +++++++++++-------- tox.ini | 1 + 16 files changed, 191 insertions(+), 74 deletions(-) create mode 100644 tests/integration/postgres/test_bootaction_signalling.py delete mode 100644 tests/yaml_samples/bootaction.yaml create mode 100644 tests/yaml_samples/deckhand_bootaction.yaml diff --git a/docs/source/bootaction.rst b/docs/source/bootaction.rst index a7d7c090..e6e0b199 100644 --- a/docs/source/bootaction.rst +++ b/docs/source/bootaction.rst @@ -18,6 +18,7 @@ for these YAML documents is described below. .. code-block:: yaml data: + signaling: true assets: - path: /save/file/here location: http://get.data.here/data @@ -35,6 +36,10 @@ for these YAML documents is described below. ... +``signaling`` is a boolean noting whether Drydock should expect a signal at the completion +of this boot action. If set to ``true`` for a boot action that does not send a signal, it +will elongate the deployment step and consider the boot action failed. + ``assets`` is a list of data assets. More details below on how each data asset is rendered. ``node_filter`` is an optional filter for selecting to which nodes this boot action will apply. @@ -98,6 +103,7 @@ template - node.labels - Key, value pairs of both explicit and dynamic labels for this node - action.key - A key that uniquely identifies this boot action on this node. Can be used for signaling boot action result. - action.report_url - The URL that can be POSTed to for reporting boot action result. + - action.design_ref - The design reference for the deployment that initiated the bootaction Also available in the Jinja2 template is the ``urlencode`` filter to encode a string for inclusion in a URL. diff --git a/drydock_provisioner/control/bootaction.py b/drydock_provisioner/control/bootaction.py index 26a6a62c..dc82a98f 100644 --- a/drydock_provisioner/control/bootaction.py +++ b/drydock_provisioner/control/bootaction.py @@ -179,6 +179,7 @@ class BootactionAssetsResource(StatefulResource): hostname, site_design, action_id, + task.design_ref, type_filter=asset_type_filter)) tarball = BootactionUtils.tarbuilder(asset_list=assets) diff --git a/drydock_provisioner/ingester/plugins/deckhand.py b/drydock_provisioner/ingester/plugins/deckhand.py index 5bde4ab9..78475281 100644 --- a/drydock_provisioner/ingester/plugins/deckhand.py +++ b/drydock_provisioner/ingester/plugins/deckhand.py @@ -396,6 +396,7 @@ class DeckhandIngester(IngesterPlugin): nfs = self.process_bootaction_nodefilter(node_filter) model.node_filter = nfs + model.signaling = data.get('signaling', None) return model def process_bootaction_asset(self, asset_dict): diff --git a/drydock_provisioner/ingester/plugins/yaml.py b/drydock_provisioner/ingester/plugins/yaml.py index f3b4bbb5..1b4234ad 100644 --- a/drydock_provisioner/ingester/plugins/yaml.py +++ b/drydock_provisioner/ingester/plugins/yaml.py @@ -396,6 +396,7 @@ class YamlIngester(IngesterPlugin): nfs = self.process_bootaction_nodefilter(node_filter) model.node_filter = nfs + model.signaling = data.get('signaling', None) return model def process_bootaction_asset(self, asset_dict): diff --git a/drydock_provisioner/objects/bootaction.py b/drydock_provisioner/objects/bootaction.py index f24a7807..400922cc 100644 --- a/drydock_provisioner/objects/bootaction.py +++ b/drydock_provisioner/objects/bootaction.py @@ -43,6 +43,8 @@ class BootAction(base.DrydockPersistentObject, base.DrydockObject): ovo_fields.ObjectField('NodeFilterSet', nullable=True), 'target_nodes': ovo_fields.ListOfStringsField(nullable=True), + 'signaling': + ovo_fields.BooleanField(default=True), } def __init__(self, **kwargs): @@ -55,7 +57,7 @@ class BootAction(base.DrydockPersistentObject, base.DrydockObject): def get_name(self): return self.name - def render_assets(self, nodename, site_design, action_id, + def render_assets(self, nodename, site_design, action_id, design_ref, type_filter=None): """Render all of the assets in this bootaction. @@ -68,13 +70,14 @@ class BootAction(base.DrydockPersistentObject, base.DrydockObject): :param site_design: a objects.SiteDesign instance holding the design sets :param action_id: a 128-bit ULID action_id of the boot action the assets are part of + :param design_ref: the design ref this boot action was initiated under :param type_filter: optional filter of the types of assets to render """ assets = list() for a in self.asset_list: if type_filter is None or (type_filter is not None and a.type == type_filter): - a.render(nodename, site_design, action_id) + a.render(nodename, site_design, action_id, design_ref) assets.append(a) return assets @@ -116,7 +119,7 @@ class BootActionAsset(base.DrydockObject): super().__init__(permissions=mode, **kwargs) self.rendered_bytes = None - def render(self, nodename, site_design, action_id): + def render(self, nodename, site_design, action_id, design_ref): """Render this asset into a base64 encoded string. The ``nodename`` and ``action_id`` will be used to construct @@ -125,6 +128,7 @@ class BootActionAsset(base.DrydockObject): :param nodename: the name of the node where the asset will be deployed :param site_design: instance of objects.SiteDesign :param action_id: a 128-bit ULID boot action id + :param design_ref: The design ref this bootaction was initiated under """ node = site_design.get_baremetal_node(nodename) @@ -139,6 +143,7 @@ class BootActionAsset(base.DrydockObject): 'action': { 'key': ulid2.ulid_to_base32(action_id), 'report_url': config.config_mgr.conf.bootactions.report_url, + 'design_ref': design_ref, } } diff --git a/drydock_provisioner/orchestrator/orchestrator.py b/drydock_provisioner/orchestrator/orchestrator.py index a1227d7b..0191e13e 100644 --- a/drydock_provisioner/orchestrator/orchestrator.py +++ b/drydock_provisioner/orchestrator/orchestrator.py @@ -546,9 +546,17 @@ class Orchestrator(object): identity_key = os.urandom(32) self.state_manager.post_boot_action_context( nodename, task.get_id(), identity_key) - action_id = ulid2.generate_binary_ulid() - self.state_manager.post_boot_action( - nodename, task.get_id(), identity_key, action_id, ba.name) + if ba.signaling: + self.logger.debug( + "Adding boot action %s for node %s to the database." % + (ba.name, nodename)) + action_id = ulid2.generate_binary_ulid() + self.state_manager.post_boot_action( + nodename, + task.get_id(), identity_key, action_id, ba.name) + else: + self.logger.debug( + "Boot action %s has disabled signaling." % ba.name) return identity_key diff --git a/drydock_provisioner/schemas/bootaction.yaml b/drydock_provisioner/schemas/bootaction.yaml index 7230d8ac..d35e6683 100644 --- a/drydock_provisioner/schemas/bootaction.yaml +++ b/drydock_provisioner/schemas/bootaction.yaml @@ -11,6 +11,8 @@ data: type: 'object' additionalProperties: false properties: + signaling: + type: 'boolean' assets: type: 'array' items: diff --git a/tests/integration/postgres/test_bootaction_signalling.py b/tests/integration/postgres/test_bootaction_signalling.py new file mode 100644 index 00000000..d9db9792 --- /dev/null +++ b/tests/integration/postgres/test_bootaction_signalling.py @@ -0,0 +1,41 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test postgres integration for task management.""" + +from drydock_provisioner.objects import fields as hd_fields + +class TestBootActionSignal(object): + def test_bootaction_signal_disable(self, deckhand_orchestrator, drydock_state, input_files): + """Test that disabled signaling omits a status entry in the DB.""" + input_file = input_files.join("deckhand_fullsite.yaml") + design_ref = "file://%s" % str(input_file) + + task = deckhand_orchestrator.create_task( + design_ref=design_ref, + action=hd_fields.OrchestratorAction.Noop) + + deckhand_orchestrator.create_bootaction_context("compute01", task) + + # In the fullsite YAML, node 'compute01' is assigned two + # bootactions - one with signaling enabled, one disabled. + # Validate these counts + + bootactions = drydock_state.get_boot_actions_for_node('compute01') + + assert len(bootactions) == 1 + + design_status, design_data = deckhand_orchestrator.get_effective_site( + design_ref=design_ref) + + assert len(design_data.bootactions) == 2 diff --git a/tests/integration/postgres/test_postgres_bootaction_status.py b/tests/integration/postgres/test_postgres_bootaction_status.py index 74c936b7..166ffe9d 100644 --- a/tests/integration/postgres/test_postgres_bootaction_status.py +++ b/tests/integration/postgres/test_postgres_bootaction_status.py @@ -20,7 +20,7 @@ import ulid2 from drydock_provisioner import objects -class TestPostgres(object): +class TestPostgresBootAction(object): def test_bootaction_post(self, populateddb, drydock_state): """Test that a boot action status can be added.""" id_key = os.urandom(32) diff --git a/tests/unit/test_bootaction_asset_render.py b/tests/unit/test_bootaction_asset_render.py index 8744b957..1baeb29e 100644 --- a/tests/unit/test_bootaction_asset_render.py +++ b/tests/unit/test_bootaction_asset_render.py @@ -20,7 +20,8 @@ import drydock_provisioner.objects as objects class TestClass(object): - def test_bootaction_render(self, input_files, deckhand_ingester, setup): + def test_bootaction_render_nodename(self, input_files, deckhand_ingester, setup): + """Test the bootaction render routine provides expected output.""" objects.register_all() input_file = input_files.join("deckhand_fullsite.yaml") @@ -33,6 +34,24 @@ class TestClass(object): ba = design_data.get_bootaction('helloworld') action_id = ulid2.generate_binary_ulid() - assets = ba.render_assets('compute01', design_data, action_id) + assets = ba.render_assets('compute01', design_data, action_id, design_ref) assert 'compute01' in assets[0].rendered_bytes.decode('utf-8') + + def test_bootaction_render_design_ref(self, input_files, deckhand_ingester, setup): + """Test the bootaction render routine provides expected output.""" + objects.register_all() + + input_file = input_files.join("deckhand_fullsite.yaml") + + design_state = DrydockState() + design_ref = "file://%s" % str(input_file) + + design_status, design_data = deckhand_ingester.ingest_data( + design_state=design_state, design_ref=design_ref) + + ba = design_data.get_bootaction('helloworld') + action_id = ulid2.generate_binary_ulid() + assets = ba.render_assets('compute01', design_data, action_id, design_ref) + + assert 'deckhand_fullsite.yaml' in assets[2].rendered_bytes.decode('utf-8') diff --git a/tests/unit/test_bootaction_tarbuilder.py b/tests/unit/test_bootaction_tarbuilder.py index 29ab79d2..2c35798c 100644 --- a/tests/unit/test_bootaction_tarbuilder.py +++ b/tests/unit/test_bootaction_tarbuilder.py @@ -39,7 +39,7 @@ class TestClass(object): ba = design_data.get_bootaction('helloworld') action_id = ulid2.generate_binary_ulid() - assets = ba.render_assets(target_host, design_data, action_id) + assets = ba.render_assets(target_host, design_data, action_id, design_ref) assert len(assets) > 0 diff --git a/tests/unit/test_ingester_bootaction.py b/tests/unit/test_ingester_bootaction.py index a5abfcce..2a252045 100644 --- a/tests/unit/test_ingester_bootaction.py +++ b/tests/unit/test_ingester_bootaction.py @@ -17,11 +17,11 @@ from drydock_provisioner.statemgmt.state import DrydockState import drydock_provisioner.objects as objects -class TestClass(object): +class TestBootAction(object): def test_bootaction_parse(self, input_files, deckhand_ingester, setup): objects.register_all() - input_file = input_files.join("bootaction.yaml") + input_file = input_files.join("deckhand_bootaction.yaml") design_state = DrydockState() design_ref = "file://%s" % str(input_file) @@ -31,4 +31,4 @@ class TestClass(object): ba = design_data.get_bootaction('helloworld') - assert len(ba.asset_list) == 2 + assert len(ba.asset_list) == 3 diff --git a/tests/yaml_samples/bootaction.yaml b/tests/yaml_samples/bootaction.yaml deleted file mode 100644 index 425d812e..00000000 --- a/tests/yaml_samples/bootaction.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -schema: 'drydock/BootAction/v1' -metadata: - schema: 'metadata/Document/v1' - name: helloworld - storagePolicy: 'cleartext' - labels: - application: 'drydock' -data: - assets: - - path: /var/tmp/hello.sh - type: file - permissions: '555' - data: |- - IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19 - Jwo= - data_pipeline: - - base64_decode - - utf8_decode - - template - - path: /lib/systemd/system/hello.service - type: unit - permissions: '600' - data: |- - W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4 - ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu - dGFyZ2V0Cg== - data_pipeline: - - base64_decode - - utf8_decode -... diff --git a/tests/yaml_samples/deckhand_bootaction.yaml b/tests/yaml_samples/deckhand_bootaction.yaml new file mode 100644 index 00000000..d1d51569 --- /dev/null +++ b/tests/yaml_samples/deckhand_bootaction.yaml @@ -0,0 +1,53 @@ +#Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +schema: 'drydock/BootAction/v1' +metadata: + schema: 'metadata/Document/v1' + name: helloworld + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + signaling: true + assets: + - path: /var/tmp/hello.sh + type: file + permissions: '555' + data: |- + IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19 + Jwo= + data_pipeline: + - base64_decode + - utf8_decode + - template + - path: /lib/systemd/system/hello.service + type: unit + permissions: '600' + data: |- + W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4 + ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu + dGFyZ2V0Cg== + data_pipeline: + - base64_decode + - utf8_decode + - path: /var/tmp/designref.sh + type: file + permissions: '500' + data: e3sgYWN0aW9uLmRlc2lnbl9yZWYgfX0K + data_pipeline: + - base64_decode + - utf8_decode + - template +... diff --git a/tests/yaml_samples/deckhand_fullsite.yaml b/tests/yaml_samples/deckhand_fullsite.yaml index 829d964f..04a5675a 100644 --- a/tests/yaml_samples/deckhand_fullsite.yaml +++ b/tests/yaml_samples/deckhand_fullsite.yaml @@ -381,36 +381,6 @@ data: bus_type: 'scsi' --- schema: 'drydock/BootAction/v1' -metadata: - schema: 'metadata/Document/v1' - name: helloworld - storagePolicy: 'cleartext' - labels: - application: 'drydock' -data: - assets: - - path: /var/tmp/hello.sh - type: file - permissions: '555' - data: |- - IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19 - Jwo= - data_pipeline: - - base64_decode - - utf8_decode - - template - - path: /lib/systemd/system/hello.service - type: unit - permissions: '600' - data: |- - W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4 - ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu - dGFyZ2V0Cg== - data_pipeline: - - base64_decode - - utf8_decode ---- -schema: 'drydock/BootAction/v1' metadata: schema: 'metadata/Document/v1' name: hw_filtered @@ -418,6 +388,7 @@ metadata: labels: application: 'drydock' data: + signaling: false node_filter: filter_set_type: 'union' filter_set: @@ -446,3 +417,42 @@ data: - base64_decode - utf8_decode ... +--- +schema: 'drydock/BootAction/v1' +metadata: + schema: 'metadata/Document/v1' + name: helloworld + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + assets: + - path: /var/tmp/hello.sh + type: file + permissions: '555' + data: |- + IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19 + Jwo= + data_pipeline: + - base64_decode + - utf8_decode + - template + - path: /lib/systemd/system/hello.service + type: unit + permissions: '600' + data: |- + W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4 + ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu + dGFyZ2V0Cg== + data_pipeline: + - base64_decode + - utf8_decode + - path: /var/tmp/designref.sh + type: file + permissions: '500' + data: e3sgYWN0aW9uLmRlc2lnbl9yZWYgfX0K + data_pipeline: + - base64_decode + - utf8_decode + - template +... diff --git a/tox.ini b/tox.ini index 848f417c..2a631925 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,7 @@ commands= passenv=DRYDOCK_IMAGE_NAME IMAGE_PREFIX IMAGE_TAG setenv= PYTHONWARNING=all + YAMLDIR={toxinidir}/tests/yaml_samples/ commands= {toxinidir}/tests/postgres/start_postgres.sh py.test \