summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorScott Hussey <sh8121@att.com>2018-11-08 16:15:32 -0600
committerScott Hussey <sh8121@att.com>2018-11-27 12:04:38 -0600
commita2418241ce82c9e2c846fc9959cc60b505c1d37e (patch)
tree15a9757cb5fc7a300660da646cc53789e2e3cf89
parent44316619866bdf741851868045677c4b78ef766a (diff)
(fix) Use endpoint for MAAS URL
- Instead of forcing a user to provide the full URL for the MAAS API, instead use the endpoints pattern and render the URI via HTK templates. - Add secret name to chart to support HTK ingress - Install libyaml to take advantage of faster parsing by pyyaml - Add exception logging when node compiling fails. - Add caching of parsed design to gain efficiency - Add TLS certificate secret for use by the ingress document Change-Id: I5a2dbc415483c336d38d67edcebdfc5812f7bb0c
Notes
Notes (review): Code-Review+2: Aaron Sheffield <ajs@sheffieldfamily.net> Code-Review+2: Mark Burnett <mark.m.burnett@gmail.com> Workflow+1: Mark Burnett <mark.m.burnett@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Tue, 04 Dec 2018 14:25:20 +0000 Reviewed-on: https://review.openstack.org/616709 Project: openstack/airship-drydock Branch: refs/heads/master
-rw-r--r--charts/drydock/templates/configmap-etc.yaml5
-rw-r--r--charts/drydock/templates/secret-tls.yaml18
-rw-r--r--charts/drydock/values.yaml38
-rw-r--r--python/drydock_provisioner/drivers/node/maasdriver/api_client.py3
-rw-r--r--python/drydock_provisioner/ingester/plugins/deckhand.py24
-rw-r--r--python/drydock_provisioner/orchestrator/orchestrator.py6
-rw-r--r--python/drydock_provisioner/statemgmt/design/resolver.py2
-rw-r--r--python/tests/unit/test_api_nodes_unit.py30
-rw-r--r--python/tests/unit/test_api_validation.py8
-rw-r--r--requirements-host.txt1
-rwxr-xr-xtools/helm_tk.sh2
11 files changed, 105 insertions, 32 deletions
diff --git a/charts/drydock/templates/configmap-etc.yaml b/charts/drydock/templates/configmap-etc.yaml
index 9575915..21d5de0 100644
--- a/charts/drydock/templates/configmap-etc.yaml
+++ b/charts/drydock/templates/configmap-etc.yaml
@@ -22,6 +22,11 @@
22{{- tuple "postgresql" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.drydock.database "database_connect_string" | quote | trunc 0 -}} 22{{- tuple "postgresql" "internal" "user" "postgresql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | set .Values.conf.drydock.database "database_connect_string" | quote | trunc 0 -}}
23{{- end -}} 23{{- end -}}
24 24
25# Render MAAS connection string if it is not explicitly configured
26{{- if empty .Values.conf.drydock.maasdriver.maas_api_url -}}
27{{- tuple "maas_region" "internal" "region_api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.maasdriver "maas_api_url" | quote | trunc 0 -}}
28{{- end -}}
29
25{{- if empty .Values.conf.drydock.keystone_authtoken.auth_uri -}} 30{{- if empty .Values.conf.drydock.keystone_authtoken.auth_uri -}}
26{{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.keystone_authtoken "auth_uri" | quote | trunc 0 -}} 31{{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.drydock.keystone_authtoken "auth_uri" | quote | trunc 0 -}}
27{{- end -}} 32{{- end -}}
diff --git a/charts/drydock/templates/secret-tls.yaml b/charts/drydock/templates/secret-tls.yaml
new file mode 100644
index 0000000..3a7cace
--- /dev/null
+++ b/charts/drydock/templates/secret-tls.yaml
@@ -0,0 +1,18 @@
1{{/*
2# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15*/}}
16{{- if .Values.manifests.secret_tls }}
17{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "physicalprovisioner" ) -}}
18{{- end }}
diff --git a/charts/drydock/values.yaml b/charts/drydock/values.yaml
index 7333657..fc0fa9a 100644
--- a/charts/drydock/values.yaml
+++ b/charts/drydock/values.yaml
@@ -131,6 +131,7 @@ manifests:
131 secret_keystone: true 131 secret_keystone: true
132 secret_database: true 132 secret_database: true
133 secret_ssh_key: true 133 secret_ssh_key: true
134 secret_tls: true
134 configmap_etc: true 135 configmap_etc: true
135 configmap_bin: true 136 configmap_bin: true
136 service_drydock: true 137 service_drydock: true
@@ -234,7 +235,12 @@ endpoints:
234 default: http 235 default: http
235 host_fqdn_override: 236 host_fqdn_override:
236 default: null 237 default: null
237 public: drydock.test.local 238 public:
239 host: drydock.test.local
240# tls:
241# crt: replace
242# ca: replace
243# key: replace
238 postgresql: 244 postgresql:
239 name: postgresql 245 name: postgresql
240 auth: 246 auth:
@@ -254,7 +260,29 @@ endpoints:
254 default: 5432 260 default: 5432
255 host_fqdn_override: 261 host_fqdn_override:
256 default: null 262 default: null
257 263 maas_region:
264 name: maas-region
265 auth:
266 admin:
267 username: admin
268 password: admin
269 email: none@none
270 hosts:
271 default: maas-region
272 public: maas
273 path:
274 default: /MAAS
275 scheme:
276 default: http
277 port:
278 region_api:
279 default: 80
280 public: 80
281 nodeport: 31900
282 region_proxy:
283 default: 8000
284 host_fqdn_override:
285 default: null
258secrets: 286secrets:
259 identity: 287 identity:
260 admin: drydock-keystone-admin 288 admin: drydock-keystone-admin
@@ -263,6 +291,10 @@ secrets:
263 admin: drydock-postgresql-admin 291 admin: drydock-postgresql-admin
264 user: drydock-postgresql-user 292 user: drydock-postgresql-user
265 ssh_key: ssh-private-key 293 ssh_key: ssh-private-key
294 tls:
295 physicalprovisioner:
296 api:
297 public: drydock-tls-public
266 298
267# Settings for drydock.conf 299# Settings for drydock.conf
268conf: 300conf:
@@ -288,7 +320,7 @@ conf:
288 control_logger_name: '${global_logger_name}.control' 320 control_logger_name: '${global_logger_name}.control'
289 maasdriver: 321 maasdriver:
290 maas_api_key: 'override_this' 322 maas_api_key: 'override_this'
291 maas_api_url: 'override_this' 323 maas_api_url: null
292 plugins: 324 plugins:
293 ingester: 325 ingester:
294 - 'drydock_provisioner.ingester.plugins.yaml.YamlIngester' 326 - 'drydock_provisioner.ingester.plugins.yaml.YamlIngester'
diff --git a/python/drydock_provisioner/drivers/node/maasdriver/api_client.py b/python/drydock_provisioner/drivers/node/maasdriver/api_client.py
index a5c48e3..8f56dc8 100644
--- a/python/drydock_provisioner/drivers/node/maasdriver/api_client.py
+++ b/python/drydock_provisioner/drivers/node/maasdriver/api_client.py
@@ -54,7 +54,8 @@ class MaasOauth(req_auth.AuthBase):
54 54
55class MaasRequestFactory(object): 55class MaasRequestFactory(object):
56 def __init__(self, base_url, apikey): 56 def __init__(self, base_url, apikey):
57 self.base_url = base_url 57 # The URL in the config should end in /MAAS/, but the api is behind /MAAS/api/2.0/
58 self.base_url = base_url + "/api/2.0/"
58 self.apikey = apikey 59 self.apikey = apikey
59 60
60 self.signer = MaasOauth(apikey) 61 self.signer = MaasOauth(apikey)
diff --git a/python/drydock_provisioner/ingester/plugins/deckhand.py b/python/drydock_provisioner/ingester/plugins/deckhand.py
index 7bf650b..4a8ae03 100644
--- a/python/drydock_provisioner/ingester/plugins/deckhand.py
+++ b/python/drydock_provisioner/ingester/plugins/deckhand.py
@@ -19,13 +19,23 @@ import jsonschema
19import os 19import os
20import pkg_resources 20import pkg_resources
21import copy 21import copy
22import hashlib
22 23
23import drydock_provisioner.objects.fields as hd_fields 24import drydock_provisioner.objects.fields as hd_fields
24 25
26from beaker.cache import CacheManager
27from beaker.util import parse_cache_config_options
28
25from drydock_provisioner import error as errors 29from drydock_provisioner import error as errors
26from drydock_provisioner import objects 30from drydock_provisioner import objects
27from drydock_provisioner.ingester.plugins import IngesterPlugin 31from drydock_provisioner.ingester.plugins import IngesterPlugin
28 32
33cache_opts = {
34 'cache.type': 'memory',
35 'expire': 1800,
36}
37
38cache = CacheManager(**parse_cache_config_options(cache_opts))
29 39
30class DeckhandIngester(IngesterPlugin): 40class DeckhandIngester(IngesterPlugin):
31 def __init__(self): 41 def __init__(self):
@@ -44,8 +54,20 @@ class DeckhandIngester(IngesterPlugin):
44 54
45 :returns: a tuple of a status response and a list of parsed objects from drydock_provisioner.objects 55 :returns: a tuple of a status response and a list of parsed objects from drydock_provisioner.objects
46 """ 56 """
57 def local_parse():
58 return self.parse_docs(kwargs.get('content'))
59
47 if 'content' in kwargs: 60 if 'content' in kwargs:
48 parse_status, models = self.parse_docs(kwargs.get('content')) 61 try:
62 # Hash the input to use as the cache key. This is not a security
63 # related hash, so use cheap and fast MD5
64 hv = hashlib.md5(kwargs.get('content', b'')).hexdigest()
65 local_cache = cache.get_cache('parsed_docs')
66 results = local_cache.get(key=hv, createfunc=local_parse)
67 parse_status, models = results
68 except Exception as ex:
69 self.logger.debug("Error parsing design - hash %s", hv, exc_info=ex)
70 raise ex
49 else: 71 else:
50 raise ValueError('Missing parameter "content"') 72 raise ValueError('Missing parameter "content"')
51 73
diff --git a/python/drydock_provisioner/orchestrator/orchestrator.py b/python/drydock_provisioner/orchestrator/orchestrator.py
index 473d1b8..863b167 100644
--- a/python/drydock_provisioner/orchestrator/orchestrator.py
+++ b/python/drydock_provisioner/orchestrator/orchestrator.py
@@ -269,10 +269,10 @@ class Orchestrator(object):
269 site_design, 269 site_design,
270 state_manager=self.state_manager, 270 state_manager=self.state_manager,
271 resolve_aliases=resolve_aliases) 271 resolve_aliases=resolve_aliases)
272 except Exception: 272 except Exception as ex:
273 node_failed.append(n) 273 node_failed.append(n)
274 self.logger.error( 274 self.logger.debug(
275 "Failed to build applied model for node %s." % n.name) 275 "Failed to build applied model for node %s.", n.name, exc_info=ex)
276 if node_failed: 276 if node_failed:
277 raise errors.DesignError( 277 raise errors.DesignError(
278 "Failed to build applied model for %s" % ",".join( 278 "Failed to build applied model for %s" % ",".join(
diff --git a/python/drydock_provisioner/statemgmt/design/resolver.py b/python/drydock_provisioner/statemgmt/design/resolver.py
index 639cdba..0feaeab 100644
--- a/python/drydock_provisioner/statemgmt/design/resolver.py
+++ b/python/drydock_provisioner/statemgmt/design/resolver.py
@@ -28,7 +28,7 @@ from drydock_provisioner.config import config_mgr
28 28
29cache_opts = { 29cache_opts = {
30 'cache.type': 'memory', 30 'cache.type': 'memory',
31 'expire': 300, 31 'expire': 180,
32} 32}
33 33
34cache = CacheManager(**parse_cache_config_options(cache_opts)) 34cache = CacheManager(**parse_cache_config_options(cache_opts))
diff --git a/python/tests/unit/test_api_nodes_unit.py b/python/tests/unit/test_api_nodes_unit.py
index 20ddbe1..76edf8e 100644
--- a/python/tests/unit/test_api_nodes_unit.py
+++ b/python/tests/unit/test_api_nodes_unit.py
@@ -13,7 +13,6 @@
13# limitations under the License. 13# limitations under the License.
14"""Test Nodes API""" 14"""Test Nodes API"""
15from falcon import testing 15from falcon import testing
16from unittest.mock import Mock
17 16
18import pytest 17import pytest
19import json 18import json
@@ -85,20 +84,15 @@ class TestNodesApiUnit(object):
85 84
86 85
87@pytest.fixture() 86@pytest.fixture()
88def mock_process_node_filter(deckhand_orchestrator): 87def mock_process_node_filter(mocker, deckhand_orchestrator):
89 def side_effect(**kwargs): 88 n1 = objects.BaremetalNode()
90 n1 = objects.BaremetalNode() 89 n1.name = 'n1'
91 n1.name = 'n1' 90 n1.site = 'test1'
92 n1.site = 'test1' 91 n2 = objects.BaremetalNode()
93 n2 = objects.BaremetalNode() 92 n2.name = 'n2'
94 n2.name = 'n2' 93 n2.site = 'test2'
95 n2.site = 'test2' 94 mock_results = [n1, n2]
96 return [n1, n2] 95
97 96 with mocker.patch('drydock_provisioner.orchestrator.orchestrator.Orchestrator.process_node_filter',
98 deckhand_orchestrator.real_process_node_filter = deckhand_orchestrator.process_node_filter 97 mocker.MagicMock(return_value=mock_results)):
99 deckhand_orchestrator.process_node_filter = Mock(side_effect=side_effect) 98 yield
100
101 yield
102 deckhand_orchestrator.process_node_filter = Mock(
103 wraps=None, side_effect=None)
104 deckhand_orchestrator.process_node_filter = deckhand_orchestrator.real_process_node_filter
diff --git a/python/tests/unit/test_api_validation.py b/python/tests/unit/test_api_validation.py
index c04fee0..f64d95e 100644
--- a/python/tests/unit/test_api_validation.py
+++ b/python/tests/unit/test_api_validation.py
@@ -27,7 +27,7 @@ LOG = logging.getLogger(__name__)
27 27
28 28
29class TestValidationApi(object): 29class TestValidationApi(object):
30 def test_post_validation_resp(self, input_files, falcontest, drydock_state, 30 def test_post_validation_resp(self, setup_logging, input_files, falcontest, drydock_state,
31 mock_get_build_data): 31 mock_get_build_data):
32 32
33 input_file = input_files.join("deckhand_fullsite.yaml") 33 input_file = input_files.join("deckhand_fullsite.yaml")
@@ -52,7 +52,7 @@ class TestValidationApi(object):
52 LOG.debug(result.text) 52 LOG.debug(result.text)
53 assert result.status == falcon.HTTP_200 53 assert result.status == falcon.HTTP_200
54 54
55 def test_href_error(self, input_files, falcontest): 55 def test_href_error(self, setup_logging, input_files, falcontest):
56 url = '/api/v1.0/validatedesign' 56 url = '/api/v1.0/validatedesign'
57 hdr = { 57 hdr = {
58 'Content-Type': 'application/json', 58 'Content-Type': 'application/json',
@@ -72,7 +72,7 @@ class TestValidationApi(object):
72 LOG.debug(result.text) 72 LOG.debug(result.text)
73 assert result.status == falcon.HTTP_400 73 assert result.status == falcon.HTTP_400
74 74
75 def test_json_data_error(self, input_files, falcontest): 75 def test_json_data_error(self, setup_logging, input_files, falcontest):
76 url = '/api/v1.0/validatedesign' 76 url = '/api/v1.0/validatedesign'
77 hdr = { 77 hdr = {
78 'Content-Type': 'application/json', 78 'Content-Type': 'application/json',
@@ -88,7 +88,7 @@ class TestValidationApi(object):
88 LOG.debug(result.text) 88 LOG.debug(result.text)
89 assert result.status == falcon.HTTP_400 89 assert result.status == falcon.HTTP_400
90 90
91 def test_invalid_post_resp(self, input_files, falcontest, drydock_state, 91 def test_invalid_post_resp(self, setup_logging, input_files, falcontest, drydock_state,
92 mock_get_build_data): 92 mock_get_build_data):
93 input_file = input_files.join("invalid_validation.yaml") 93 input_file = input_files.join("invalid_validation.yaml")
94 design_ref = "file://%s" % str(input_file) 94 design_ref = "file://%s" % str(input_file)
diff --git a/requirements-host.txt b/requirements-host.txt
index ee59c62..39087e5 100644
--- a/requirements-host.txt
+++ b/requirements-host.txt
@@ -1,6 +1,7 @@
1# These are host packages needed for Drydock 1# These are host packages needed for Drydock
2# that don't come on a minimal Ubuntu install 2# that don't come on a minimal Ubuntu install
3libvirt-dev 3libvirt-dev
4libyaml-dev
4pkg-config 5pkg-config
5python3-dev 6python3-dev
6gcc 7gcc
diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh
index 1a1a366..b8d2fe0 100755
--- a/tools/helm_tk.sh
+++ b/tools/helm_tk.sh
@@ -18,7 +18,7 @@
18HELM=$1 18HELM=$1
19HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"} 19HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"}
20HTK_PATH=${HTK_PATH:-""} 20HTK_PATH=${HTK_PATH:-""}
21HTK_STABLE_COMMIT=${HTK_COMMIT:-"2fce7e821201a5f578331370703e164d5a932fcc"} 21HTK_STABLE_COMMIT=${HTK_COMMIT:-"42249d4243d892b7d04ce6aed9b0c8d7edcbfc7a"}
22DEP_UP_LIST=${DEP_UP_LIST:-"drydock"} 22DEP_UP_LIST=${DEP_UP_LIST:-"drydock"}
23BUILD_DIR=${BUILD_DIR:-$(mktemp -d)} 23BUILD_DIR=${BUILD_DIR:-$(mktemp -d)}
24 24