(fix) Use endpoint for MAAS URL

- Instead of forcing a user to provide the full URL for the MAAS API,
  instead use the endpoints pattern and render the URI via HTK templates.
- Add secret name to chart to support HTK ingress
- Install libyaml to take advantage of faster parsing by pyyaml
- Add exception logging when node compiling fails.
- Add caching of parsed design to gain efficiency
- Add TLS certificate secret for use by the ingress document

Change-Id: I5a2dbc415483c336d38d67edcebdfc5812f7bb0c
This commit is contained in:
Scott Hussey 2018-11-08 16:15:32 -06:00
parent 4431661986
commit a2418241ce
11 changed files with 104 additions and 31 deletions

View File

@ -22,6 +22,11 @@
{{- tuple "postgresql" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.drydock.database "database_connect_string" | quote | trunc 0 -}}
{{- end -}}
# Render MAAS connection string if it is not explicitly configured
{{- if empty .Values.conf.drydock.maasdriver.maas_api_url -}}
{{- tuple "maas_region" "internal" "region_api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.maasdriver "maas_api_url" | quote | trunc 0 -}}
{{- end -}}
{{- if empty .Values.conf.drydock.keystone_authtoken.auth_uri -}}
{{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.keystone_authtoken "auth_uri" | quote | trunc 0 -}}
{{- end -}}

View File

@ -0,0 +1,18 @@
{{/*
# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/}}
{{- if .Values.manifests.secret_tls }}
{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "physicalprovisioner" ) -}}
{{- end }}

View File

@ -131,6 +131,7 @@ manifests:
secret_keystone: true
secret_database: true
secret_ssh_key: true
secret_tls: true
configmap_etc: true
configmap_bin: true
service_drydock: true
@ -234,7 +235,12 @@ endpoints:
default: http
host_fqdn_override:
default: null
public: drydock.test.local
public:
host: drydock.test.local
# tls:
# crt: replace
# ca: replace
# key: replace
postgresql:
name: postgresql
auth:
@ -254,7 +260,29 @@ endpoints:
default: 5432
host_fqdn_override:
default: null
maas_region:
name: maas-region
auth:
admin:
username: admin
password: admin
email: none@none
hosts:
default: maas-region
public: maas
path:
default: /MAAS
scheme:
default: http
port:
region_api:
default: 80
public: 80
nodeport: 31900
region_proxy:
default: 8000
host_fqdn_override:
default: null
secrets:
identity:
admin: drydock-keystone-admin
@ -263,6 +291,10 @@ secrets:
admin: drydock-postgresql-admin
user: drydock-postgresql-user
ssh_key: ssh-private-key
tls:
physicalprovisioner:
api:
public: drydock-tls-public
# Settings for drydock.conf
conf:
@ -288,7 +320,7 @@ conf:
control_logger_name: '${global_logger_name}.control'
maasdriver:
maas_api_key: 'override_this'
maas_api_url: 'override_this'
maas_api_url: null
plugins:
ingester:
- 'drydock_provisioner.ingester.plugins.yaml.YamlIngester'

View File

@ -54,7 +54,8 @@ class MaasOauth(req_auth.AuthBase):
class MaasRequestFactory(object):
def __init__(self, base_url, apikey):
self.base_url = base_url
# The URL in the config should end in /MAAS/, but the api is behind /MAAS/api/2.0/
self.base_url = base_url + "/api/2.0/"
self.apikey = apikey
self.signer = MaasOauth(apikey)

View File

@ -19,13 +19,23 @@ import jsonschema
import os
import pkg_resources
import copy
import hashlib
import drydock_provisioner.objects.fields as hd_fields
from beaker.cache import CacheManager
from beaker.util import parse_cache_config_options
from drydock_provisioner import error as errors
from drydock_provisioner import objects
from drydock_provisioner.ingester.plugins import IngesterPlugin
cache_opts = {
'cache.type': 'memory',
'expire': 1800,
}
cache = CacheManager(**parse_cache_config_options(cache_opts))
class DeckhandIngester(IngesterPlugin):
def __init__(self):
@ -44,8 +54,20 @@ class DeckhandIngester(IngesterPlugin):
:returns: a tuple of a status response and a list of parsed objects from drydock_provisioner.objects
"""
def local_parse():
return self.parse_docs(kwargs.get('content'))
if 'content' in kwargs:
parse_status, models = self.parse_docs(kwargs.get('content'))
try:
# Hash the input to use as the cache key. This is not a security
# related hash, so use cheap and fast MD5
hv = hashlib.md5(kwargs.get('content', b'')).hexdigest()
local_cache = cache.get_cache('parsed_docs')
results = local_cache.get(key=hv, createfunc=local_parse)
parse_status, models = results
except Exception as ex:
self.logger.debug("Error parsing design - hash %s", hv, exc_info=ex)
raise ex
else:
raise ValueError('Missing parameter "content"')

View File

@ -269,10 +269,10 @@ class Orchestrator(object):
site_design,
state_manager=self.state_manager,
resolve_aliases=resolve_aliases)
except Exception:
except Exception as ex:
node_failed.append(n)
self.logger.error(
"Failed to build applied model for node %s." % n.name)
self.logger.debug(
"Failed to build applied model for node %s.", n.name, exc_info=ex)
if node_failed:
raise errors.DesignError(
"Failed to build applied model for %s" % ",".join(

View File

@ -28,7 +28,7 @@ from drydock_provisioner.config import config_mgr
cache_opts = {
'cache.type': 'memory',
'expire': 300,
'expire': 180,
}
cache = CacheManager(**parse_cache_config_options(cache_opts))

View File

@ -13,7 +13,6 @@
# limitations under the License.
"""Test Nodes API"""
from falcon import testing
from unittest.mock import Mock
import pytest
import json
@ -85,20 +84,15 @@ class TestNodesApiUnit(object):
@pytest.fixture()
def mock_process_node_filter(deckhand_orchestrator):
def side_effect(**kwargs):
n1 = objects.BaremetalNode()
n1.name = 'n1'
n1.site = 'test1'
n2 = objects.BaremetalNode()
n2.name = 'n2'
n2.site = 'test2'
return [n1, n2]
def mock_process_node_filter(mocker, deckhand_orchestrator):
n1 = objects.BaremetalNode()
n1.name = 'n1'
n1.site = 'test1'
n2 = objects.BaremetalNode()
n2.name = 'n2'
n2.site = 'test2'
mock_results = [n1, n2]
deckhand_orchestrator.real_process_node_filter = deckhand_orchestrator.process_node_filter
deckhand_orchestrator.process_node_filter = Mock(side_effect=side_effect)
yield
deckhand_orchestrator.process_node_filter = Mock(
wraps=None, side_effect=None)
deckhand_orchestrator.process_node_filter = deckhand_orchestrator.real_process_node_filter
with mocker.patch('drydock_provisioner.orchestrator.orchestrator.Orchestrator.process_node_filter',
mocker.MagicMock(return_value=mock_results)):
yield

View File

@ -27,7 +27,7 @@ LOG = logging.getLogger(__name__)
class TestValidationApi(object):
def test_post_validation_resp(self, input_files, falcontest, drydock_state,
def test_post_validation_resp(self, setup_logging, input_files, falcontest, drydock_state,
mock_get_build_data):
input_file = input_files.join("deckhand_fullsite.yaml")
@ -52,7 +52,7 @@ class TestValidationApi(object):
LOG.debug(result.text)
assert result.status == falcon.HTTP_200
def test_href_error(self, input_files, falcontest):
def test_href_error(self, setup_logging, input_files, falcontest):
url = '/api/v1.0/validatedesign'
hdr = {
'Content-Type': 'application/json',
@ -72,7 +72,7 @@ class TestValidationApi(object):
LOG.debug(result.text)
assert result.status == falcon.HTTP_400
def test_json_data_error(self, input_files, falcontest):
def test_json_data_error(self, setup_logging, input_files, falcontest):
url = '/api/v1.0/validatedesign'
hdr = {
'Content-Type': 'application/json',
@ -88,7 +88,7 @@ class TestValidationApi(object):
LOG.debug(result.text)
assert result.status == falcon.HTTP_400
def test_invalid_post_resp(self, input_files, falcontest, drydock_state,
def test_invalid_post_resp(self, setup_logging, input_files, falcontest, drydock_state,
mock_get_build_data):
input_file = input_files.join("invalid_validation.yaml")
design_ref = "file://%s" % str(input_file)

View File

@ -1,6 +1,7 @@
# These are host packages needed for Drydock
# that don't come on a minimal Ubuntu install
libvirt-dev
libyaml-dev
pkg-config
python3-dev
gcc

View File

@ -18,7 +18,7 @@
HELM=$1
HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"}
HTK_PATH=${HTK_PATH:-""}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"2fce7e821201a5f578331370703e164d5a932fcc"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"42249d4243d892b7d04ce6aed9b0c8d7edcbfc7a"}
DEP_UP_LIST=${DEP_UP_LIST:-"drydock"}
BUILD_DIR=${BUILD_DIR:-$(mktemp -d)}