airship
/
drydock
Code Issues Proposed changes

Merge remote-tracking branch 'attcomdev/master' 

Resolve conflicts
This commit is contained in:
Scott Hussey 2017-07-12 12:49:22 -05:00
parent 273f11f21a 91d18d1004
commit a11d55327e
16 changed files with 362 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
Dockerfile
View File

@ -13,16 +13,36 @@
# limitations under the License.
FROM drydock_base:0.1
ARG VERSION
ENV DEBIAN_FRONTEND noninteractive
ENV container docker
ADD drydock.conf /etc/drydock/drydock.conf
ADD . /tmp/drydock
RUN apt -qq update && \
apt -y install git \
netbase \
python3-minimal \
python3-setuptools \
python3-pip \
python3-dev \
ca-certificates \
gcc \
g++ \
make \
libffi-dev \
libssl-dev --no-install-recommends
# Copy direct dependency requirements only to build a dependency layer
COPY ./requirements-direct.txt /tmp/drydock/
RUN pip3 install -r /tmp/drydock/requirements-direct.txt
COPY . /tmp/drydock
WORKDIR /tmp/drydock
RUN python3 setup.py install
EXPOSE 9000
CMD ["uwsgi","--http",":9000","-w","drydock_provisioner.drydock","--callable","drydock","--enable-threads","-L","--python-autoreload","1","--pyargv","--config-file /etc/drydock/drydock.conf"]
ENTRYPOINT ["./entrypoint.sh"]
CMD ["server"]

1
drydock_provisioner/config.py
View File

@ -22,7 +22,6 @@ package. It is assumed that:
returns a dict where:
* The keys are strings which are the group names.
* The value of each key is a list of config options for that group.
* The conf package doesn't have further packages with config options.

2
drydock_provisioner/control/base.py
View File

@ -117,7 +117,7 @@ class DrydockRequestContext(object):
self.log_level = 'ERROR'
self.user = None
self.roles = ['anyone']
self.req_id = str(uuid.uuid4())
self.request_id = str(uuid.uuid4())
self.external_marker = None
def set_log_level(self, level):

1
drydock_provisioner/control/middleware.py
View File

@ -73,7 +73,6 @@ class ContextMiddleware(object):
ctx.set_log_level('INFO')
ext_marker = req.get_header('X-Context-Marker')
ctx.set_external_marker(ext_marker if ext_marker is not None else '')
class LoggingMiddleware(object):

2
drydock_provisioner/drivers/node/__init__.py
View File

@ -56,4 +56,4 @@ class NodeDriver(ProviderDriver):

217
drydock_provisioner/drivers/node/maasdriver/driver.py
View File

@ -122,6 +122,7 @@ class MaasNodeDriver(NodeDriver):
site_design = self.orchestrator.get_effective_site(design_id)
if task.action == hd_fields.OrchestratorAction.CreateNetworkTemplate:
self.orchestrator.task_field_update(task.get_id(), status=hd_fields.TaskStatus.Running)
subtask = self.orchestrator.create_task(task_model.DriverTask,
@ -143,13 +144,16 @@ class MaasNodeDriver(NodeDriver):
'retry': False,
'detail': 'MaaS Network creation timed-out'
}
self.logger.warning("Thread for task %s timed out after 120s" % (subtask.get_id()))
self.orchestrator.task_field_update(task.get_id(),
status=hd_fields.TaskStatus.Complete,
result=hd_fields.ActionResult.Failure,
result_detail=result)
else:
subtask = self.state_manager.get_task(subtask.get_id())
self.logger.info("Thread for task %s completed - result %s" % (subtask.get_id(), subtask.get_result()))
self.orchestrator.task_field_update(task.get_id(),
status=hd_fields.TaskStatus.Complete,
@ -566,6 +570,213 @@ class MaasNodeDriver(NodeDriver):
status=hd_fields.TaskStatus.Complete,
result=result,
result_detail=result_detail)
elif task.action == hd_fields.OrchestratorAction.ApplyNodeNetworking:
self.orchestrator.task_field_update(task.get_id(),
status=hd_fields.TaskStatus.Running)
self.logger.debug("Starting subtask to configure networking on %s nodes." % (len(task.node_list)))
subtasks = []
result_detail = {
'detail': [],
'failed_nodes': [],
'successful_nodes': [],
}
for n in task.node_list:
subtask = self.orchestrator.create_task(task_model.DriverTask,
parent_task_id=task.get_id(), design_id=design_id,
action=hd_fields.OrchestratorAction.ApplyNodeNetworking,
site_name=task.site_name,
task_scope={'site': task.site_name, 'node_names': [n]})
runner = MaasTaskRunner(state_manager=self.state_manager,
orchestrator=self.orchestrator,
task_id=subtask.get_id())
self.logger.info("Starting thread for task %s to configure networking on node %s" % (subtask.get_id(), n))
runner.start()
subtasks.append(subtask.get_id())
running_subtasks = len(subtasks)
attempts = 0
worked = failed = False
while running_subtasks > 0 and attempts < cfg.CONF.timeouts.apply_node_networking:
for t in subtasks:
subtask = self.state_manager.get_task(t)
if subtask.status == hd_fields.TaskStatus.Complete:
self.logger.info("Task %s to apply networking on node %s complete - status %s" %
(subtask.get_id(), n, subtask.get_result()))
running_subtasks = running_subtasks - 1
if subtask.result == hd_fields.ActionResult.Success:
result_detail['successful_nodes'].extend(subtask.node_list)
worked = True
elif subtask.result == hd_fields.ActionResult.Failure:
result_detail['failed_nodes'].extend(subtask.node_list)
failed = True
elif subtask.result == hd_fields.ActionResult.PartialSuccess:
worked = failed = True
time.sleep(1 * 60)
attempts = attempts + 1
if running_subtasks > 0:
self.logger.warning("Time out for task %s before all subtask threads complete" % (task.get_id()))
result = hd_fields.ActionResult.DependentFailure
result_detail['detail'].append('Some subtasks did not complete before the timeout threshold')
elif worked and failed:
result = hd_fields.ActionResult.PartialSuccess
elif worked:
result = hd_fields.ActionResult.Success
else:
result = hd_fields.ActionResult.Failure
self.orchestrator.task_field_update(task.get_id(),
status=hd_fields.TaskStatus.Complete,
result=result,
result_detail=result_detail)
elif task.action == hd_fields.OrchestratorAction.ApplyNodePlatform:
self.orchestrator.task_field_update(task.get_id(),
status=hd_fields.TaskStatus.Running)
self.logger.debug("Starting subtask to configure the platform on %s nodes." % (len(task.node_list)))
subtasks = []
result_detail = {
'detail': [],
'failed_nodes': [],
'successful_nodes': [],
}
for n in task.node_list:
subtask = self.orchestrator.create_task(task_model.DriverTask,
parent_task_id=task.get_id(), design_id=design_id,
action=hd_fields.OrchestratorAction.ApplyNodePlatform,
site_name=task.site_name,
task_scope={'site': task.site_name, 'node_names': [n]})
runner = MaasTaskRunner(state_manager=self.state_manager,
orchestrator=self.orchestrator,
task_id=subtask.get_id())
self.logger.info("Starting thread for task %s to config node %s platform" % (subtask.get_id(), n))
runner.start()
subtasks.append(subtask.get_id())
running_subtasks = len(subtasks)
attempts = 0
worked = failed = False
while running_subtasks > 0 and attempts < drydock_provisioner.conf.timeouts.apply_node_platform:
for t in subtasks:
subtask = self.state_manager.get_task(t)
if subtask.status == hd_fields.TaskStatus.Complete:
self.logger.info("Task %s to configure node %s platform complete - status %s" %
(subtask.get_id(), n, subtask.get_result()))
running_subtasks = running_subtasks - 1
if subtask.result == hd_fields.ActionResult.Success:
result_detail['successful_nodes'].extend(subtask.node_list)
worked = True
elif subtask.result == hd_fields.ActionResult.Failure:
result_detail['failed_nodes'].extend(subtask.node_list)
failed = True
elif subtask.result == hd_fields.ActionResult.PartialSuccess:
worked = failed = True
time.sleep(1 * 60)
attempts = attempts + 1
if running_subtasks > 0:
self.logger.warning("Time out for task %s before all subtask threads complete" % (task.get_id()))
result = hd_fields.ActionResult.DependentFailure
result_detail['detail'].append('Some subtasks did not complete before the timeout threshold')
elif worked and failed:
result = hd_fields.ActionResult.PartialSuccess
elif worked:
result = hd_fields.ActionResult.Success
else:
result = hd_fields.ActionResult.Failure
self.orchestrator.task_field_update(task.get_id(),
status=hd_fields.TaskStatus.Complete,
result=result,
result_detail=result_detail)
elif task.action == hd_fields.OrchestratorAction.DeployNode:
self.orchestrator.task_field_update(task.get_id(),
status=hd_fields.TaskStatus.Running)
self.logger.debug("Starting subtask to deploy %s nodes." % (len(task.node_list)))
subtasks = []
result_detail = {
'detail': [],
'failed_nodes': [],
'successful_nodes': [],
}
for n in task.node_list:
subtask = self.orchestrator.create_task(task_model.DriverTask,
parent_task_id=task.get_id(), design_id=design_id,
action=hd_fields.OrchestratorAction.DeployNode,
site_name=task.site_name,
task_scope={'site': task.site_name, 'node_names': [n]})
runner = MaasTaskRunner(state_manager=self.state_manager,
orchestrator=self.orchestrator,
task_id=subtask.get_id())
self.logger.info("Starting thread for task %s to deploy node %s" % (subtask.get_id(), n))
runner.start()
subtasks.append(subtask.get_id())
running_subtasks = len(subtasks)
attempts = 0
worked = failed = False
while running_subtasks > 0 and attempts < cfg.CONF.timeouts.deploy_node:
for t in subtasks:
subtask = self.state_manager.get_task(t)
if subtask.status == hd_fields.TaskStatus.Complete:
self.logger.info("Task %s to deploy node %s complete - status %s" %
(subtask.get_id(), n, subtask.get_result()))
running_subtasks = running_subtasks - 1
if subtask.result == hd_fields.ActionResult.Success:
result_detail['successful_nodes'].extend(subtask.node_list)
worked = True
elif subtask.result == hd_fields.ActionResult.Failure:
result_detail['failed_nodes'].extend(subtask.node_list)
failed = True
elif subtask.result == hd_fields.ActionResult.PartialSuccess:
worked = failed = True
time.sleep(1 * 60)
attempts = attempts + 1
if running_subtasks > 0:
self.logger.warning("Time out for task %s before all subtask threads complete" % (task.get_id()))
result = hd_fields.ActionResult.DependentFailure
result_detail['detail'].append('Some subtasks did not complete before the timeout threshold')
elif worked and failed:
result = hd_fields.ActionResult.PartialSuccess
elif worked:
result = hd_fields.ActionResult.Success
else:
result = hd_fields.ActionResult.Failure
self.orchestrator.task_field_update(task.get_id(),
status=hd_fields.TaskStatus.Complete,
result=result,
result_detail=result_detail)
class MaasTaskRunner(drivers.DriverTaskRunner):
@ -1168,7 +1379,7 @@ class MaasTaskRunner(drivers.DriverTaskRunner):
# Render the string of all kernel params for the node
kp_string = ""
for k,v in getattr(node, 'kernel_params', {}).items():
for k, v in getattr(node, 'kernel_params', {}).items():
if v == 'True':
kp_string = kp_string + " %s" % (k)
else:
@ -1232,7 +1443,9 @@ class MaasTaskRunner(drivers.DriverTaskRunner):
self.logger.error("Error configuring static tags for node %s: %s" % (node.name, str(ex3)))
continue
if failed:
if worked and failed:
final_result = hd_fields.ActionResult.PartialSuccess
elif failed:
final_result = hd_fields.ActionResult.Failure
else:
final_result = hd_fields.ActionResult.Success

1
drydock_provisioner/drivers/node/maasdriver/models/base.py
View File

@ -145,7 +145,6 @@ class ResourceBase(object):
return i
class ResourceCollectionBase(object):
"""
A collection of MaaS resources.

2
drydock_provisioner/drivers/node/maasdriver/models/machine.py
View File

@ -282,4 +282,4 @@ class Machines(model_base.ResourceCollectionBase):
return res
raise errors.DriverError("Failed updating MAAS url %s - return code %s"
% (url, resp.status_code))
% (url, resp.status_code))

37
drydock_provisioner/drivers/node/maasdriver/models/sshkey.py Normal file
View File

@ -0,0 +1,37 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import drydock_provisioner.error as errors
import drydock_provisioner.drivers.node.maasdriver.models.base as model_base
class SshKey(model_base.ResourceBase):
resource_url = 'account/prefs/sshkeys/{resource_id}/'
fields = ['resource_id', 'key']
json_fields = ['key']
def __init__(self, api_client, **kwargs):
super(SshKey, self).__init__(api_client, **kwargs)
#Keys should never have newlines, but sometimes they get added
self.key = self.key.replace("\n","")
class SshKeys(model_base.ResourceCollectionBase):
collection_url = 'account/prefs/sshkeys/'
collection_resource = SshKey
def __init__(self, api_client, **kwargs):
super(SshKeys, self).__init__(api_client)

6
drydock_provisioner/drivers/node/maasdriver/models/tag.py
View File

@ -42,7 +42,9 @@ class Tag(model_base.ResourceBase):
system_id_list = []
for n in resp_json:
system_id_list.append(n.get('system_id'))
system_id = n.get('system_id', None)
if system_id is not None:
system_id_list.append(system_id)
return system_id_list
else:
@ -100,7 +102,7 @@ class Tag(model_base.ResourceBase):
refined_dict = {k: obj_dict.get(k, None) for k in cls.fields}
if 'name' in obj_dict.keys():
if 'name' in obj_dict:
refined_dict['resource_id'] = obj_dict.get('name')
i = cls(api_client, **refined_dict)

3
drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py
View File

@ -39,6 +39,8 @@ class PyghmiDriver(oob.OobDriver):
driver_key = "pyghmi_driver"
driver_desc = "Pyghmi OOB Driver"
oob_types_supported = ['ipmi']
def __init__(self, **kwargs):
super(PyghmiDriver, self).__init__(**kwargs)
@ -176,7 +178,6 @@ class PyghmiTaskRunner(drivers.DriverTaskRunner):
raise errors.DriverError("Runner node does not match " \
"task node scope")
self.orchestrator.task_field_update(self.task.get_id(),
status=hd_fields.TaskStatus.Running)

17
drydock_provisioner/drydock.py
View File

@ -12,10 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
<<<<<<< HEAD
import sys
import os
from oslo_config import cfg
=======
from oslo_config import cfg
import sys
>>>>>>> attcomdev/master
import drydock_provisioner.config as config
import drydock_provisioner.objects as objects
@ -37,7 +42,11 @@ def start_drydock():
cfg.CONF(sys.argv[1:])
if cfg.CONF.debug:
<<<<<<< HEAD
cfg.CONF.set_override(name='log_level', override='DEBUG', group='logging')
=======
cfg.CONF.logging.log_level = 'DEBUG'
>>>>>>> attcomdev/master
# Setup root logger
logger = logging.getLogger(cfg.CONF.logging.global_logger_name)
@ -63,6 +72,7 @@ def start_drydock():
input_ingester = ingester.Ingester()
input_ingester.enable_plugins(cfg.CONF.plugins.ingester)
<<<<<<< HEAD
# Check if we have an API key in the environment
# Hack around until we move MaaS configs to the YAML schema
if 'MAAS_API_KEY' in os.environ:
@ -75,6 +85,13 @@ def start_drydock():
cfg.CONF.log_opt_values(logging.getLogger(cfg.CONF.logging.global_logger_name), logging.DEBUG)
return wsgi_callable
=======
# Now that loggers are configured, log the effective config
cfg.CONF.log_opt_values(logging.getLogger(cfg.CONF.logging.global_logger_name), logging.DEBUG)
# Now that loggers are configured, log the effective config
drydock_provisioner.conf.log_opt_values(logging.getLogger(drydock_provisioner.conf.logging.global_logger_name), logging.DEBUG)
>>>>>>> attcomdev/master
drydock = start_drydock()

40
drydock_provisioner/ingester/plugins/yaml.py
View File

@ -70,9 +70,7 @@ class YamlIngester(IngesterPlugin):
"""
def parse_docs(self, yaml_string):
models = []
self.logger.debug("yamlingester:parse_docs - Parsing YAML string \n%s" % (yaml_string))
try:
parsed_data = yaml.load_all(yaml_string)
except yaml.YAMLError as err:
@ -274,10 +272,19 @@ class YamlIngester(IngesterPlugin):
storage = spec.get('storage', {})
model.storage_layout = storage.get('layout', 'lvm')
<<<<<<< HEAD
bootdisk = storage.get('bootdisk', {})
model.bootdisk_device = bootdisk.get('device', None)
model.bootdisk_root_size = bootdisk.get('root_size', None)
model.bootdisk_boot_size = bootdisk.get('boot_size', None)
=======
model.oob_parameters = {}
for k,v in oob.items():
if k == 'type':
model.oob_type = oob.get('type', None)
else:
model.oob_parameters[k] = v
>>>>>>> attcomdev/master
partitions = storage.get('partitions', [])
model.partitions = objects.HostPartitionList()
@ -310,8 +317,13 @@ class YamlIngester(IngesterPlugin):
int_model.hardware_slaves = []
slaves = i.get('slaves', [])
<<<<<<< HEAD
for s in slaves:
int_model.hardware_slaves.append(s)
=======
int_model.device_name = i.get('device_name', None)
int_model.network_link = i.get('device_link', None)
>>>>>>> attcomdev/master
int_model.networks = []
networks = i.get('networks', [])
@ -335,7 +347,24 @@ class YamlIngester(IngesterPlugin):
node_metadata = spec.get('metadata', {})
metadata_tags = node_metadata.get('tags', [])
<<<<<<< HEAD
model.tags = [t for t in metadata_tags]
=======
platform = spec.get('platform', {})
model.image = platform.get('image', None)
model.kernel = platform.get('kernel', None)
model.kernel_params = {}
for k, v in platform.get('kernel_params', {}).items():
model.kernel_params[k] = v
model.primary_network = spec.get('primary_network', None)
node_metadata = spec.get('metadata', {})
metadata_tags = node_metadata.get('tags', [])
model.tags = []
>>>>>>> attcomdev/master
owner_data = node_metadata.get('owner_data', {})
model.owner_data = {}
@ -348,7 +377,14 @@ class YamlIngester(IngesterPlugin):
if kind == 'BaremetalNode':
model.boot_mac = node_metadata.get('boot_mac', None)
<<<<<<< HEAD
addresses = spec.get('addressing', [])
=======
if kind == 'BaremetalNode':
model.boot_mac = node_metadata.get('boot_mac', None)
addresses = spec.get('addressing', [])
>>>>>>> attcomdev/master
if len(addresses) == 0:
raise ValueError('BaremetalNode needs at least' \

3
drydock_provisioner/orchestrator/__init__.py
View File

@ -349,7 +349,6 @@ class Orchestrator(object):
design_id=design_id,
action=hd_fields.OrchestratorAction.SetNodeBoot,
task_scope=task_scope)
self.logger.info("Starting OOB driver task %s to set PXE boot for OOB type %s" %
(setboot_task.get_id(), oob_type))
@ -454,7 +453,6 @@ class Orchestrator(object):
final_result = hd_fields.ActionResult.Success
else:
final_result = hd_fields.ActionResult.Failure
self.task_field_update(task_id,
status=hd_fields.TaskStatus.Complete,
@ -553,7 +551,6 @@ class Orchestrator(object):
else:
self.logger.warning("No nodes successfully networked, skipping platform configuration subtask")
final_result = None
if worked and failed:
final_result = hd_fields.ActionResult.PartialSuccess

12
entrypoint.sh Executable file
View File

@ -0,0 +1,12 @@
#!/bin/bash
CMD="drydock"
PORT="8000"
set -e
if [ "$1" = 'server' ]; then
exec uwsgi --http :${PORT} -w drydock_provisioner.drydock --callable drydock --enable-threads -L --pyargv "--config-file /etc/drydock/drydock.conf"
fi
exec ${CMD} $@

10
requirements-direct.txt Normal file
View File

@ -0,0 +1,10 @@
PyYAML
pyghmi>=1.0.18
netaddr
falcon
oslo.versionedobjects>=1.23.0
requests
oauthlib
uwsgi>1.4
bson===0.4.7
oslo.config