diff --git a/drydock_provisioner/drivers/node/maasdriver/actions/node.py b/drydock_provisioner/drivers/node/maasdriver/actions/node.py index 0847b26b..5ad5f05e 100644 --- a/drydock_provisioner/drivers/node/maasdriver/actions/node.py +++ b/drydock_provisioner/drivers/node/maasdriver/actions/node.py @@ -17,6 +17,7 @@ import time import logging import re import math +import yaml from datetime import datetime @@ -2049,11 +2050,32 @@ class DeployNode(BaseMaasAction): "Error setting boot action id key tag for %s." % n.name, exc_info=ex) + # Extract bootaction assets that are package lists as they + # are included in the deployment initiation + + node_packages = self.orchestrator.find_node_package_lists( + n.name, self.task) + user_data_dict = dict(packages=[]) + + for k, v in node_packages.items(): + if v: + user_data_dict['packages'].append([k, v]) + else: + user_data_dict['packages'].append(k) + + user_data_string = None + if user_data_dict.get('packages'): + user_data_string = "#cloud-config\n%s" % yaml.dump( + user_data_dict) + self.logger.info("Deploying node %s: image=%s, kernel=%s" % (n.name, n.image, n.kernel)) try: - machine.deploy(platform=n.image, kernel=n.kernel) + machine.deploy( + platform=n.image, + kernel=n.kernel, + user_data=user_data_string) except errors.DriverError: msg = "Error deploying node %s, skipping" % n.name self.logger.warning(msg) diff --git a/drydock_provisioner/drivers/node/maasdriver/models/machine.py b/drydock_provisioner/drivers/node/maasdriver/models/machine.py index 662a88a8..2e0a5eb1 100644 --- a/drydock_provisioner/drivers/node/maasdriver/models/machine.py +++ b/drydock_provisioner/drivers/node/maasdriver/models/machine.py @@ -13,6 +13,7 @@ # limitations under the License. """Model representing MAAS node/machine resource.""" import logging +import base64 import drydock_provisioner.error as errors import drydock_provisioner.drivers.node.maasdriver.models.base as model_base @@ -250,14 +251,15 @@ class Machine(model_base.ResourceBase): def deploy(self, user_data=None, platform=None, kernel=None): """Start the MaaS deployment process. - :param user_data: cloud-init user data + :param user_data: ``str`` of cloud-init user data :param platform: Which image to install :param kernel: Which kernel to enable """ deploy_options = {} if user_data is not None: - deploy_options['user_data'] = user_data + deploy_options['user_data'] = base64.b64encode( + user_data.encode('utf-8')).decode('utf-8') if platform is not None: deploy_options['distro_series'] = platform diff --git a/drydock_provisioner/objects/bootaction.py b/drydock_provisioner/objects/bootaction.py index f472e069..7ba36594 100644 --- a/drydock_provisioner/objects/bootaction.py +++ b/drydock_provisioner/objects/bootaction.py @@ -106,7 +106,7 @@ class BootActionAsset(base.DrydockObject): VERSION = '1.0' fields = { - 'type': ovo_fields.StringField(nullable=True), + 'type': hd_fields.BootactionAssetTypeField(nullable=True), 'path': ovo_fields.StringField(nullable=True), 'location': ovo_fields.StringField(nullable=True), 'data': ovo_fields.StringField(nullable=True), @@ -127,7 +127,7 @@ class BootActionAsset(base.DrydockObject): ba_type = kwargs.get('type', None) package_list = None - if ba_type == 'pkg_list': + if ba_type == hd_fields.BootactionAssetType.PackageList: if isinstance(kwargs.get('data'), dict): package_list = self._extract_package_list(kwargs.pop('data')) # If the data section doesn't parse as a dictionary @@ -158,12 +158,12 @@ class BootActionAsset(base.DrydockObject): rendered_location = self.execute_pipeline( self.location, self.location_pipeline, tpl_ctx=tpl_ctx) data_block = self.resolve_asset_location(rendered_location) - if self.type == 'pkg_list': + if self.type == hd_fields.BootactionAssetType.PackageList: self._parse_package_list(data_block) - elif self.type != 'pkg_list': + elif self.type != hd_fields.BootactionAssetType.PackageList: data_block = self.data.encode('utf-8') - if self.type != 'pkg_list': + if self.type != hd_fields.BootactionAssetType.PackageList: value = self.execute_pipeline( data_block, self.data_pipeline, tpl_ctx=tpl_ctx) diff --git a/drydock_provisioner/objects/fields.py b/drydock_provisioner/objects/fields.py index 954cf5f3..8f261529 100644 --- a/drydock_provisioner/objects/fields.py +++ b/drydock_provisioner/objects/fields.py @@ -205,3 +205,14 @@ class MessageLevels(BaseDrydockEnum): class DocumentType(BaseDrydockEnum): Deckhand = 'deckhand' + + +class BootactionAssetType(BaseDrydockEnum): + PackageList = "pkg_list" + Unit = "unit" # SystemD Unit + File = "file" + + ALL = (PackageList, Unit, File) + +class BootactionAssetTypeField(fields.BaseEnumField): + AUTO_TYPE = BootactionAssetType() diff --git a/drydock_provisioner/orchestrator/orchestrator.py b/drydock_provisioner/orchestrator/orchestrator.py index 31bdd444..ad74c970 100644 --- a/drydock_provisioner/orchestrator/orchestrator.py +++ b/drydock_provisioner/orchestrator/orchestrator.py @@ -595,6 +595,35 @@ class Orchestrator(object): action_status=init_status) return identity_key + def find_node_package_lists(self, nodename, task): + """Return all packages to be installed on ``nodename`` + + :param nodename: The name of the node to retrieve packages for + :param task: The task initiating this request + """ + design_status, site_design = self.get_effective_site(task.design_ref) + + if site_design.bootactions is None: + return None + + self.logger.debug( + "Extracting package install list for node %s" % nodename) + + pkg_list = dict() + + for ba in site_design.bootactions: + if nodename in ba.target_nodes: + assets = ba.render_assets( + nodename, + site_design, + ulid2.generate_binary_ulid(), + task.design_ref, + type_filter=hd_fields.BootactionAssetType.PackageList) + for a in assets: + pkg_list.update(a.package_list) + + return pkg_list + def render_route_domains(self, site_design): """Update site_design with static routes for route domains. diff --git a/drydock_provisioner/statemgmt/design/resolver.py b/drydock_provisioner/statemgmt/design/resolver.py index 0160ae76..0ab7cf6a 100644 --- a/drydock_provisioner/statemgmt/design/resolver.py +++ b/drydock_provisioner/statemgmt/design/resolver.py @@ -18,15 +18,24 @@ import re import logging import requests +from beaker.cache import CacheManager +from beaker.util import parse_cache_config_options from drydock_provisioner import error as errors from drydock_provisioner.util import KeystoneUtils +cache_opts = { + 'cache.type': 'memory', + 'expire': 300, +} + +cache = CacheManager(**parse_cache_config_options(cache_opts)) class ReferenceResolver(object): """Class for handling different data references to resolve them data.""" @classmethod + @cache.cache() def resolve_reference(cls, design_ref): """Resolve a reference to a design document. diff --git a/requirements-direct.txt b/requirements-direct.txt index b54e1306..11daf662 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -23,3 +23,4 @@ jinja2==2.9.6 ulid2==0.1.1 defusedxml===0.5.0 libvirt-python==3.10.0 +beaker==1.9.1 diff --git a/requirements-lock.txt b/requirements-lock.txt index bee5be9d..efdc2ae1 100644 --- a/requirements-lock.txt +++ b/requirements-lock.txt @@ -1,7 +1,8 @@ alembic==0.8.2 amqp==2.2.2 Babel==2.5.3 -cachetools==2.0.1 +Beaker==1.9.1 +cachetools==2.1.0 certifi==2018.4.16 chardet==3.0.4 click==6.7 @@ -19,7 +20,7 @@ Jinja2==2.9.6 jsonschema==2.6.0 keystoneauth1==3.3.0 keystonemiddleware==4.9.1 -kombu==4.1.0 +kombu==4.2.0 libvirt-python==3.10.0 Mako==1.0.7 MarkupSafe==1.0 @@ -27,7 +28,7 @@ monotonic==1.5 msgpack==0.5.6 netaddr==0.7.19 netifaces==0.10.7 -oauthlib==2.0.7 +oauthlib==2.1.0 oslo.concurrency==3.27.0 oslo.config==5.2.0 oslo.context==2.20.0 @@ -37,12 +38,12 @@ oslo.messaging==6.2.0 oslo.middleware==3.35.0 oslo.policy==1.22.1 oslo.serialization==2.25.0 -oslo.service==1.31.1 -oslo.utils==3.36.1 +oslo.service==1.31.2 +oslo.utils==3.36.2 oslo.versionedobjects==1.23.0 Paste==2.0.3 PasteDeploy==1.5.2 -pbr==4.0.2 +pbr==4.0.3 pip==10.0.1 positional==1.2.1 prettytable==0.7.2 @@ -63,7 +64,7 @@ repoze.lru==0.7 requests==2.18.4 rfc3986==1.1.0 Routes==2.4.1 -setuptools==39.1.0 +setuptools==39.2.0 six==1.11.0 SQLAlchemy==1.1.14 statsd==3.2.2 @@ -74,5 +75,5 @@ urllib3==1.22 uWSGI==2.0.15 vine==1.1.4 WebOb==1.8.1 -wheel==0.31.0 +wheel==0.31.1 wrapt==1.10.11 diff --git a/tox.ini b/tox.ini index 837f50b0..7dbad836 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ deps= -rrequirements-direct.txt commands= rm requirements-lock.txt - sh -c "pip freeze --all | grep -v 'drydock-provisioner|pyinotify|pkg-resources==0.0.0' > requirements-lock.txt" + sh -c "pip freeze --all | grep -vE 'drydock-provisioner|pyinotify|pkg-resources==0.0.0' > requirements-lock.txt" [testenv:yapf] basepython=python3