[411430] Validate bootaction pkg_list

- Add a validator for bootactions to warn if a node doesn't
  have at least one
- Add a validator for bootactions to error if a package
  version specifier is invalid
- Unit tests for the validation

Change-Id: I61d8aa3831791af0484498e6fe9f7c1c83dbf540
This commit is contained in:
Scott Hussey 2018-05-07 16:12:52 -05:00
parent 1b0797440b
commit e35712a573
10 changed files with 418 additions and 83 deletions

View File

@ -69,3 +69,4 @@ Topology Documentation
:maxdepth: 1 :maxdepth: 1
topology topology
troubleshooting/index

View File

@ -0,0 +1,40 @@
..
Copyright 2017 AT&T Intellectual Property.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
=============================
Dryodck Troubleshooting Guide
=============================
This is a guide for troubleshooting issues that can arise when either
installing/deploying Drydock or using Drydock to deploy nodes.
Deployment Troubleshooting
--------------------------
Under Construction
Topology Validation
-------------------
.. toctree::
:maxdepth: 2
validations
Node Deployment
---------------
Under Construction

View File

@ -0,0 +1,39 @@
..
Copyright 2018 AT&T Intellectual Property.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
=============================
Dryodck Topology Validation
=============================
DD1XXX - Storage Validations
=============================
To be continued
DD2XXX - Network Validations
=============================
To be continued
DD3XXX - Platform Validations
=============================
To be continued
DD4XXX - Bootaction Validations
=============================
To be continued

View File

@ -50,6 +50,8 @@ class BootAction(base.DrydockPersistentObject, base.DrydockObject):
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
if not self.target_nodes:
self.target_nodes = []
# NetworkLink keyed by name # NetworkLink keyed by name
def get_id(self): def get_id(self):
@ -123,9 +125,11 @@ class BootActionAsset(base.DrydockObject):
mode = None mode = None
ba_type = kwargs.get('type', None) ba_type = kwargs.get('type', None)
package_list = None
if ba_type == 'pkg_list': if ba_type == 'pkg_list':
if isinstance(kwargs.get('data'), dict): if isinstance(kwargs.get('data'), dict):
self._extract_package_list(kwargs.pop('data')) package_list = self._extract_package_list(kwargs.pop('data'))
# If the data section doesn't parse as a dictionary # If the data section doesn't parse as a dictionary
# then the package data needs to be sourced dynamically # then the package data needs to be sourced dynamically
# Otherwise the Bootaction is invalid # Otherwise the Bootaction is invalid
@ -133,7 +137,7 @@ class BootActionAsset(base.DrydockObject):
raise errors.InvalidPackageListFormat( raise errors.InvalidPackageListFormat(
"Requires a top-level mapping/object.") "Requires a top-level mapping/object.")
super().__init__(permissions=mode, **kwargs) super().__init__(package_list=package_list, permissions=mode, **kwargs)
self.rendered_bytes = None self.rendered_bytes = None
def render(self, nodename, site_design, action_id, design_ref): def render(self, nodename, site_design, action_id, design_ref):
@ -180,7 +184,7 @@ class BootActionAsset(base.DrydockObject):
parsed_data = yaml.safe_load(data_string) parsed_data = yaml.safe_load(data_string)
if isinstance(parsed_data, dict): if isinstance(parsed_data, dict):
self._extract_package_list(parsed_data) self.package_list = self._extract_package_list(parsed_data)
else: else:
raise errors.InvalidPackageListFormat( raise errors.InvalidPackageListFormat(
"Package data should have a top-level mapping/object.") "Package data should have a top-level mapping/object.")
@ -193,13 +197,14 @@ class BootActionAsset(base.DrydockObject):
:param pkg_dict: a dictionary of packages to install :param pkg_dict: a dictionary of packages to install
""" """
self.package_list = dict() package_list = dict()
for k, v in pkg_dict.items(): for k, v in pkg_dict.items():
if isinstance(k, str) and isinstance(v, str): if (isinstance(k, str) and (not v or isinstance(v, str))):
self.package_list[k] = v package_list[k] = v
else: else:
raise errors.InvalidPackageListFormat( raise errors.InvalidPackageListFormat(
"Keys and values must be strings.") "Keys and values must be strings.")
return package_list
def _get_template_context(self, nodename, site_design, action_id, def _get_template_context(self, nodename, site_design, action_id,
design_ref): design_ref):

View File

@ -0,0 +1,88 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from drydock_provisioner import error as errors
from drydock_provisioner.orchestrator.validations.validators import Validators
class BootactionDefined(Validators):
"""Issue warnings if no bootactions are defined for a node."""
def __init__(self):
super().__init__('Bootaction Definition', 'DD4001')
def run_validation(self, site_design, orchestrator=None):
"""Validate each node has at least one bootaction."""
node_list = site_design.baremetal_nodes or []
ba_list = site_design.bootactions or []
nodes_with_ba = [n for ba in ba_list for n in ba.target_nodes]
nodes_wo_ba = [n for n in node_list if n.name not in nodes_with_ba]
for n in nodes_wo_ba:
msg = "Node %s is not in scope for any bootactions." % n.name
self.report_warn(msg, [
n.doc_ref
], "It is expected all nodes have at least one post-deploy action."
)
return
class BootactionPackageListValid(Validators):
"""Check that bootactions with pkg_list assets are valid."""
def __init__(self):
super().__init__('Bootaction pkg_list Validation', 'DD4002')
version_fields = '(\d+:)?([a-zA-Z0-9.+~-]+)(-[a-zA-Z0-9.+~]+)'
self.version_fields = re.compile(version_fields)
def run_validation(self, site_design, orchestrator=None):
"""Validate that each package list in bootaction assets is valid."""
ba_list = site_design.bootactions or []
for ba in ba_list:
for a in ba.asset_list:
if a.type == 'pkg_list':
if not a.location and not a.package_list:
msg = "Bootaction has asset of type 'pkg_list' but no valid package data"
self.report_error(msg, [
ba.doc_ref
], "pkg_list bootaction assets must specify a list of packages."
)
elif a.package_list:
for p, v in a.package_list.items():
try:
self.validate_package_version(v)
except errors.InvalidPackageListFormat as ex:
msg = str(ex)
self.report_error(msg, [
ba.doc_ref
], "pkg_list version specifications must be in a valid format."
)
return
def validate_package_version(self, v):
"""Validate that a version specification is valid.
:param v: a string package specification.
"""
if not v:
return
spec_match = self.version_fields.fullmatch(v)
if not spec_match:
raise errors.InvalidPackageListFormat(
"Version specifier %s is not a valid format." % v)

View File

@ -30,6 +30,8 @@ from drydock_provisioner.orchestrator.validations.unique_network_check import Un
from drydock_provisioner.orchestrator.validations.hostname_validity import HostnameValidity from drydock_provisioner.orchestrator.validations.hostname_validity import HostnameValidity
from drydock_provisioner.orchestrator.validations.oob_valid_ipmi import IpmiValidity from drydock_provisioner.orchestrator.validations.oob_valid_ipmi import IpmiValidity
from drydock_provisioner.orchestrator.validations.oob_valid_libvirt import LibvirtValidity from drydock_provisioner.orchestrator.validations.oob_valid_libvirt import LibvirtValidity
from drydock_provisioner.orchestrator.validations.bootaction_validity import BootactionDefined
from drydock_provisioner.orchestrator.validations.bootaction_validity import BootactionPackageListValid
class Validator(): class Validator():
@ -91,5 +93,7 @@ rule_set = [
UniqueNetworkCheck(), UniqueNetworkCheck(),
HostnameValidity(), HostnameValidity(),
IpmiValidity(), IpmiValidity(),
LibvirtValidity() LibvirtValidity(),
BootactionDefined(),
BootactionPackageListValid(),
] ]

View File

@ -0,0 +1,92 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test Validation Rule Rational Boot Storage"""
import logging
from drydock_provisioner.orchestrator.orchestrator import Orchestrator
from drydock_provisioner import objects
from drydock_provisioner.objects import fields as hd_fields
from drydock_provisioner.orchestrator.validations.bootaction_validity import BootactionDefined
LOG = logging.getLogger(__name__)
class TestBootactionsValidity(object):
def test_valid_bootaction(self, deckhand_ingester, drydock_state, setup,
input_files, mock_get_build_data):
input_file = input_files.join("validation.yaml")
design_ref = "file://%s" % str(input_file)
orch = Orchestrator(
state_manager=drydock_state, ingester=deckhand_ingester)
status, site_design = Orchestrator.get_effective_site(orch, design_ref)
assert status.status == hd_fields.ValidationResult.Success
LOG.debug("%s" % status.to_dict())
validator = BootactionDefined()
message_list = validator.execute(site_design, orchestrator=orch)
msg = message_list[0].to_dict()
assert msg.get('error') is False
assert msg.get('level') == hd_fields.MessageLevels.INFO
assert len(message_list) == 1
def test_absent_bootaction(self, deckhand_ingester, drydock_state, setup,
input_files, mock_get_build_data):
input_file = input_files.join("absent_bootaction.yaml")
design_ref = "file://%s" % str(input_file)
orch = Orchestrator(
state_manager=drydock_state, ingester=deckhand_ingester)
status, site_design = Orchestrator.get_effective_site(orch, design_ref)
ba_msg = [
msg for msg in status.message_list
if ("DD4001" in msg.name
and msg.level == hd_fields.MessageLevels.WARN)
]
assert len(ba_msg) > 0
def test_invalid_bootaction_pkg_list(self, deckhand_ingester,
drydock_state, setup, input_files,
mock_get_build_data):
input_file = input_files.join("invalid_bootaction_pkg.yaml")
design_ref = "file://%s" % str(input_file)
orch = Orchestrator(
state_manager=drydock_state, ingester=deckhand_ingester)
status, site_design = Orchestrator.get_effective_site(orch, design_ref)
ba_doc = objects.DocumentReference(
doc_type=hd_fields.DocumentType.Deckhand,
doc_name="invalid_pkg_list",
doc_schema="drydock/BootAction/v1")
assert status.status == hd_fields.ValidationResult.Failure
ba_msg = [msg for msg in status.message_list if ba_doc in msg.docs]
assert len(ba_msg) > 0
for msg in ba_msg:
LOG.debug(msg)
assert ":) is not a valid format" in msg.message
assert msg.error

View File

@ -0,0 +1,128 @@
#Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
schema: 'drydock/HostProfile/v1'
metadata:
schema: 'metadata/Document/v1'
name: defaults
storagePolicy: 'cleartext'
labels:
application: 'drydock'
data:
hardware_profile: HPGen9v3
oob:
type: ipmi
network: oob
account: admin
credential: admin
storage:
physical_devices:
sda:
labels:
role: rootdisk
partitions:
- name: root
size: 20g
bootable: true
filesystem:
mountpoint: '/'
fstype: 'ext4'
mount_options: 'defaults'
- name: boot
size: 1g
bootable: false
filesystem:
mountpoint: '/boot'
fstype: 'ext4'
mount_options: 'defaults'
sdb:
volume_group: 'log_vg'
volume_groups:
log_vg:
logical_volumes:
- name: 'log_lv'
size: '500m'
filesystem:
mountpoint: '/var/log'
fstype: 'xfs'
mount_options: 'defaults'
platform:
image: 'xenial'
kernel: 'ga-16.04'
kernel_params:
quiet: true
console: ttyS2
metadata:
owner_data:
foo: bar
---
schema: 'drydock/BaremetalNode/v1'
metadata:
schema: 'metadata/Document/v1'
name: controller01
storagePolicy: 'cleartext'
labels:
application: 'drydock'
data:
host_profile: defaults
addressing:
- network: pxe
address: dhcp
- network: mgmt
address: 172.16.1.20
- network: public
address: 172.16.3.20
- network: oob
address: 172.16.100.20
metadata:
rack: rack1
---
schema: 'drydock/HardwareProfile/v1'
metadata:
schema: 'metadata/Document/v1'
name: HPGen9v3
storagePolicy: 'cleartext'
labels:
application: 'drydock'
data:
vendor: HP
generation: '8'
hw_version: '3'
bios_version: '2.2.3'
boot_mode: bios
bootstrap_protocol: pxe
pxe_interface: 0
device_aliases:
prim_nic01:
address: '0000:00:03.0'
dev_type: '82540EM Gigabit Ethernet Controller'
bus_type: 'pci'
prim_nic02:
address: '0000:00:04.0'
dev_type: '82540EM Gigabit Ethernet Controller'
bus_type: 'pci'
primary_boot:
address: '2:0.0.0'
dev_type: 'VBOX HARDDISK'
bus_type: 'scsi'
cpu_sets:
sriov: '2,4'
hugepages:
sriov:
size: '1G'
count: 300
dpdk:
size: '2M'
count: 530000
...

View File

@ -346,80 +346,4 @@ data:
dpdk: dpdk:
size: '2M' size: '2M'
count: 530000 count: 530000
---
schema: 'drydock/BootAction/v1'
metadata:
schema: 'metadata/Document/v1'
name: hw_filtered
storagePolicy: 'cleartext'
labels:
application: 'drydock'
data:
signaling: false
node_filter:
filter_set_type: 'union'
filter_set:
- filter_type: 'union'
node_names:
- 'compute01'
assets:
- path: /var/tmp/hello.sh
type: file
permissions: '555'
data: |-
IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19
Jwo=
data_pipeline:
- base64_decode
- utf8_decode
- template
- path: /lib/systemd/system/hello.service
type: unit
permissions: '600'
data: |-
W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4
ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu
dGFyZ2V0Cg==
data_pipeline:
- base64_decode
- utf8_decode
...
---
schema: 'drydock/BootAction/v1'
metadata:
schema: 'metadata/Document/v1'
name: helloworld
storagePolicy: 'cleartext'
labels:
application: 'drydock'
data:
assets:
- path: /var/tmp/hello.sh
type: file
permissions: '555'
data: |-
IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19
Jwo=
data_pipeline:
- base64_decode
- utf8_decode
- template
- path: /lib/systemd/system/hello.service
type: unit
permissions: '600'
data: |-
W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4
ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu
dGFyZ2V0Cg==
data_pipeline:
- base64_decode
- utf8_decode
- path: /var/tmp/designref.sh
type: file
permissions: '500'
data: e3sgYWN0aW9uLmRlc2lnbl9yZWYgfX0K
data_pipeline:
- base64_decode
- utf8_decode
- template
... ...

View File

@ -0,0 +1,14 @@
---
schema: 'drydock/BootAction/v1'
metadata:
schema: 'metadata/Document/v1'
name: invalid_pkg_list
storagePolicy: 'cleartext'
labels:
application: 'drydock'
data:
assets:
- type: 'pkg_list'
data:
foo: ':)'
...