From b4d95964682a9dac7d99fc3c595075ad0c1e343f Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Thu, 2 Nov 2017 14:37:14 -0500 Subject: [PATCH] Add join-scripts endpoint * Adds initial join-scripts API * Updates resiliency test to leverage API for joining Change-Id: Ibe0d42b8f4f4a3e1f6f102dee85a22cb8f78f8ec --- docs/source/api.rst | 2 +- promenade/builder.py | 23 ++++--- promenade/config.py | 15 +++++ promenade/control/api.py | 2 + promenade/control/join_scripts.py | 95 +++++++++++++++++++++++++++++ promenade/renderer.py | 6 ++ requirements-direct.txt | 3 +- requirements-frozen.txt | 10 ++- tools/g2/lib/all.sh | 1 + tools/g2/lib/nginx.sh | 18 ++++++ tools/g2/manifests/conformance.json | 38 +++++++++--- tools/g2/manifests/resiliency.json | 38 +++++++++--- tools/g2/manifests/two.json | 50 --------------- tools/g2/manifests/ucp.json | 57 ----------------- tools/g2/stages/build-scripts.sh | 2 + tools/g2/stages/gate-setup.sh | 4 ++ tools/g2/stages/join-masters.sh | 27 -------- tools/g2/stages/join-nodes.sh | 77 +++++++++++++++++++++++ tools/g2/stages/teardown-nodes.sh | 51 ++++++++++++++++ tools/stop_gate.sh | 1 + 20 files changed, 361 insertions(+), 159 deletions(-) create mode 100644 promenade/control/join_scripts.py create mode 100644 tools/g2/lib/nginx.sh delete mode 100644 tools/g2/manifests/two.json delete mode 100644 tools/g2/manifests/ucp.json delete mode 100755 tools/g2/stages/join-masters.sh create mode 100755 tools/g2/stages/join-nodes.sh create mode 100755 tools/g2/stages/teardown-nodes.sh diff --git a/docs/source/api.rst b/docs/source/api.rst index 1a4c6cd8..ac66ea5d 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -38,7 +38,7 @@ static.labels Used to set configuration options in the generated script Responses -- 204 No Content: Scripts generated successfully +- 200 OK: Script returned as response body - 400 Bad Request: One or more query parameters is missing or misspelled diff --git a/promenade/builder.py b/promenade/builder.py index 336069f2..47e686c2 100644 --- a/promenade/builder.py +++ b/promenade/builder.py @@ -80,6 +80,16 @@ class Builder: def build_node(self, node_document, *, output_dir): node_name = node_document['metadata']['name'] LOG.info('Building script for node %s', node_name) + script = self.build_node_script(node_name) + + _write_script(output_dir, _join_name(node_name), script) + + if self.validators: + validate_script = self._build_node_validate_script(node_name) + _write_script(output_dir, 'validate-%s.sh' % node_name, + validate_script) + + def build_node_script(self, node_name): sub_config = self.config.extract_node_config(node_name) file_spec_paths = [ f['path'] for f in self.config.get_path('HostSystem:files', []) @@ -88,20 +98,17 @@ class Builder: tarball = renderer.build_tarball_from_roles( config=sub_config, roles=['common', 'join'], file_specs=file_specs) - script = renderer.render_template( + return renderer.render_template( sub_config, template='scripts/join.sh', context={ 'tarball': tarball }) - _write_script(output_dir, _join_name(node_name), script) - - if self.validators: - validate_script = renderer.render_template( - sub_config, template='scripts/validate-join.sh') - _write_script(output_dir, 'validate-%s.sh' % node_name, - validate_script) + def _build_node_validate_script(self, node_name): + sub_config = self.config.extract_node_config(node_name) + return renderer.render_template( + sub_config, template='scripts/validate-join.sh') def _fetch_tar_content(*, url, path): diff --git a/promenade/config.py b/promenade/config.py index bec4deda..29680c75 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -2,6 +2,7 @@ from . import exceptions, logging, validation import copy import jinja2 import jsonpath_ng +import requests import yaml __all__ = ['Configuration'] @@ -32,6 +33,16 @@ class Configuration: return cls(documents=documents, **kwargs) + @classmethod + def from_design_ref(cls, design_ref): + response = requests.get(design_ref) + response.raise_for_status() + + documents = list(yaml.safe_load_all(response.text)) + validation.check_schemas(documents) + + return cls(documents=documents) + def __getitem__(self, path): value = self.get_path(path) if value: @@ -125,6 +136,10 @@ class Configuration: return data return default + def append(self, item): + validation.check_schema(item) + self.documents.append(item) + def _matches_filter(document, *, schema, labels): matches = True diff --git a/promenade/control/api.py b/promenade/control/api.py index 5824618c..d4dc4420 100644 --- a/promenade/control/api.py +++ b/promenade/control/api.py @@ -16,6 +16,7 @@ import falcon from promenade.control.base import BaseResource, PromenadeRequest from promenade.control.health_api import HealthResource +from promenade.control.join_scripts import JoinScriptsResource from promenade.control.middleware import (AuthMiddleware, ContextMiddleware, LoggingMiddleware) from promenade import exceptions as exc @@ -37,6 +38,7 @@ def start_api(): v1_0_routes = [ # API for managing region data ('/health', HealthResource()), + ('/join-scripts', JoinScriptsResource()), ] # Set up the 1.0 routes diff --git a/promenade/control/join_scripts.py b/promenade/control/join_scripts.py new file mode 100644 index 00000000..304464c7 --- /dev/null +++ b/promenade/control/join_scripts.py @@ -0,0 +1,95 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from promenade.control.base import BaseResource +from promenade.builder import Builder +from promenade.config import Configuration +from promenade import logging +import falcon +import kubernetes +import random + +LOG = logging.getLogger(__name__) + + +class JoinScriptsResource(BaseResource): + """ + Lists the versions supported by this API + """ + + def on_get(self, req, resp): + design_ref = req.get_param('design_ref', required=True) + ip = req.get_param('ip', required=True) + hostname = req.get_param('hostname', required=True) + + dynamic_labels = _get_param_list(req, 'labels.dynamic') + static_labels = _get_param_list(req, 'labels.static') + + join_ip = _get_join_ip() + + config = Configuration.from_design_ref(design_ref) + node_document = { + 'schema': 'promenade/KubernetesNode/v1', + 'metadata': { + 'name': hostname, + 'schema': 'metadata/Document/v1', + 'layeringDefinition': { + 'abstract': False, + 'layer': 'site' + }, + }, + 'data': { + 'hostname': hostname, + 'ip': ip, + 'join_ip': join_ip, + 'labels': { + 'dynamic': dynamic_labels, + 'static': static_labels, + }, + }, + } + config.append(node_document) + + builder = Builder(config) + script = builder.build_node_script(hostname) + + resp.body = script + resp.content_type = 'text/x-shellscript' + resp.status = falcon.HTTP_200 + + +def _get_join_ip(): + # TODO(mark-burnett): Handle errors + kubernetes.config.load_incluster_config() + client = kubernetes.client.CoreV1Api() + response = client.list_node(label_selector='kubernetes-apiserver=enabled') + + # Ignore bandit false positive: B311:blacklist + # The choice of which master to join to is a load-balancing concern, not a + # security concern. + return random.choice(list(map(_extract_ip, response.items))) # nosec + + +def _extract_ip(item): + for address in item.status.addresses: + if address.type == 'InternalIP': + return address.address + + +def _get_param_list(req, name): + values = req.get_param_as_list(name) + if values: + return values + else: + return [] diff --git a/promenade/renderer.py b/promenade/renderer.py index 46988865..b002a4ea 100644 --- a/promenade/renderer.py +++ b/promenade/renderer.py @@ -33,6 +33,8 @@ def insert_charts_into_bundler(bundler): for root, _dirnames, filenames in os.walk( '/opt/promenade/charts', followlinks=True): for source_filename in filenames: + if _source_file_is_excluded(source_filename): + continue source_path = os.path.join(root, source_filename) destination_path = os.path.join('etc/genesis/armada/assets/charts', os.path.relpath( @@ -139,6 +141,10 @@ def _default_no_proxy(network_config): return ','.join(include) +def _source_file_is_excluded(source_filename): + return source_filename.endswith('.tgz') + + def _yaml_safe_dump_all(documents): f = io.StringIO() yaml.safe_dump_all(documents, f) diff --git a/requirements-direct.txt b/requirements-direct.txt index 286a2850..e285f20e 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -4,7 +4,8 @@ jinja2==2.9.6 jsonpath-ng==1.4.3 jsonschema==2.6.0 keystonemiddleware==4.17.0 -oslo.context==2.14.0 +kubernetes==3.0.0 +oslo.context==2.19.2 pastedeploy==1.5.2 pbr==3.0.1 pyyaml==3.12 diff --git a/requirements-frozen.txt b/requirements-frozen.txt index d0c373b9..d4d1547f 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,24 +1,28 @@ Babel==2.5.1 +cachetools==2.0.1 certifi==2017.7.27.1 chardet==3.0.4 click==6.7 debtcollector==1.18.0 decorator==4.1.2 falcon==1.2.0 +google-auth==1.2.0 idna==2.6 +ipaddress==1.0.18 iso8601==0.1.12 Jinja2==2.9.6 jsonpath-ng==1.4.3 jsonschema==2.6.0 keystoneauth1==3.2.0 keystonemiddleware==4.17.0 +kubernetes==3.0.0 MarkupSafe==1.0 monotonic==1.4 msgpack-python==0.4.8 netaddr==0.7.19 netifaces==0.10.6 oslo.config==5.0.0 -oslo.context==2.14.0 +oslo.context==2.19.2 oslo.i18n==3.18.0 oslo.log==3.32.0 oslo.serialization==2.21.2 @@ -27,6 +31,8 @@ PasteDeploy==1.5.2 pbr==3.0.1 ply==3.10 positional==1.2.1 +pyasn1==0.3.7 +pyasn1-modules==0.1.5 pycadf==2.6.0 pyinotify==0.9.6 pyparsing==2.2.0 @@ -37,9 +43,11 @@ pytz==2017.3 PyYAML==3.12 requests==2.18.4 rfc3986==1.1.0 +rsa==3.4.2 six==1.11.0 stevedore==1.27.1 urllib3==1.22 uWSGI==2.0.15 WebOb==1.7.3 +websocket-client==0.40.0 wrapt==1.10.11 diff --git a/tools/g2/lib/all.sh b/tools/g2/lib/all.sh index 51d295db..c990b977 100644 --- a/tools/g2/lib/all.sh +++ b/tools/g2/lib/all.sh @@ -8,6 +8,7 @@ source "$LIB_DIR"/const.sh source "$LIB_DIR"/etcd.sh source "$LIB_DIR"/kube.sh source "$LIB_DIR"/log.sh +source "$LIB_DIR"/nginx.sh source "$LIB_DIR"/promenade.sh source "$LIB_DIR"/registry.sh source "$LIB_DIR"/ssh.sh diff --git a/tools/g2/lib/nginx.sh b/tools/g2/lib/nginx.sh new file mode 100644 index 00000000..0de7a75e --- /dev/null +++ b/tools/g2/lib/nginx.sh @@ -0,0 +1,18 @@ +nginx_down() { + REGISTRY_ID=$(docker ps -qa -f name=promenade-nginx) + if [ "x${REGISTRY_ID}" != "x" ]; then + log Removing nginx server + docker rm -fv "${REGISTRY_ID}" &>> "${LOG_FILE}" + fi +} + +nginx_up() { + log Starting nginx server to serve configuration files + mkdir -p "${TEMP_DIR}/nginx" + docker run -d \ + -p 7777:80 \ + --restart=always \ + --name promenade-nginx \ + -v "${TEMP_DIR}/nginx:/usr/share/nginx/html:ro" \ + nginx:stable &>> "${LOG_FILE}" +} diff --git a/tools/g2/manifests/conformance.json b/tools/g2/manifests/conformance.json index ed2bc500..5cca5d3a 100644 --- a/tools/g2/manifests/conformance.json +++ b/tools/g2/manifests/conformance.json @@ -29,18 +29,42 @@ }, { "name": "Join Masters", - "script": "join-masters.sh", + "script": "join-nodes.sh", "arguments": [ - "n1", - "n2", - "n3" + "-v", "n0", + "-n", "n1", + "-n", "n2", + "-n", "n3", + "-l", "calico-etcd=enabled", + "-l", "kubernetes-apiserver=enabled", + "-l", "kubernetes-controller-manager=enabled", + "-l", "kubernetes-etcd=enabled", + "-l", "kubernetes-scheduler=enabled", + "-l", "ucp-control-plane=enabled", + "-e", "kubernetes n0 genesis n1 n2 n3", + "-e", "calico n0 n0 n1 n2 n3" ] }, { - "name": "Reprovision Genesis", - "script": "reprovision-genesis.sh", + "name": "Teardown Genesis", + "script": "teardown-nodes.sh", "arguments": [ - "n1 n2 n3" + "-v", "n1", + "-n", "n0", + "-r", + "-e", "kubernetes n1 n1 n2 n3", + "-e", "calico n1 n1 n2 n3" + ] + }, + { + "name": "Join n0 as Worker", + "script": "join-nodes.sh", + "arguments": [ + "-v", "n1", + "-n", "n0", + "-l", "ucp-control-plane=enabled", + "-e", "kubernetes n1 n1 n2 n3", + "-e", "calico n1 n1 n2 n3" ] }, { diff --git a/tools/g2/manifests/resiliency.json b/tools/g2/manifests/resiliency.json index 2e8dbd7f..d8a0af85 100644 --- a/tools/g2/manifests/resiliency.json +++ b/tools/g2/manifests/resiliency.json @@ -29,18 +29,42 @@ }, { "name": "Join Masters", - "script": "join-masters.sh", + "script": "join-nodes.sh", "arguments": [ - "n1", - "n2", - "n3" + "-v", "n0", + "-n", "n1", + "-n", "n2", + "-n", "n3", + "-l", "calico-etcd=enabled", + "-l", "kubernetes-apiserver=enabled", + "-l", "kubernetes-controller-manager=enabled", + "-l", "kubernetes-etcd=enabled", + "-l", "kubernetes-scheduler=enabled", + "-l", "ucp-control-plane=enabled", + "-e", "kubernetes n0 genesis n1 n2 n3", + "-e", "calico n0 n0 n1 n2 n3" ] }, { - "name": "Reprovision Genesis", - "script": "reprovision-genesis.sh", + "name": "Teardown Genesis", + "script": "teardown-nodes.sh", "arguments": [ - "n1 n2 n3" + "-v", "n1", + "-n", "n0", + "-r", + "-e", "kubernetes n1 n1 n2 n3", + "-e", "calico n1 n1 n2 n3" + ] + }, + { + "name": "Join n0 as Worker", + "script": "join-nodes.sh", + "arguments": [ + "-v", "n1", + "-n", "n0", + "-l", "ucp-control-plane=enabled", + "-e", "kubernetes n1 n1 n2 n3", + "-e", "calico n1 n1 n2 n3" ] }, { diff --git a/tools/g2/manifests/two.json b/tools/g2/manifests/two.json deleted file mode 100644 index dab64796..00000000 --- a/tools/g2/manifests/two.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "configuration": [ - "examples/basic" - ], - "stages": [ - { - "name": "Gate Setup", - "script": "gate-setup.sh" - }, - { - "name": "Build Image", - "script": "build-image.sh" - }, - { - "name": "Generate Certificates", - "script": "generate-certificates.sh" - }, - { - "name": "Build Scripts", - "script": "build-scripts.sh" - }, - { - "name": "Create VMs", - "script": "create-vms.sh" - }, - { - "name": "Genesis", - "script": "genesis.sh" - }, - { - "name": "Join Masters", - "script": "join-masters.sh", - "arguments": [ - "n1" - ] - }, - { - "name": "Hard Reboot Cluster", - "script": "hard-reboot-cluster.sh" - } - ], - "vm": { - "memory": 2048, - "names": [ - "n0", - "n1" - ], - "vcpus": 2 - } -} diff --git a/tools/g2/manifests/ucp.json b/tools/g2/manifests/ucp.json deleted file mode 100644 index b6dcb793..00000000 --- a/tools/g2/manifests/ucp.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "configuration": [ - "examples/complete" - ], - "stages": [ - { - "name": "Gate Setup", - "script": "gate-setup.sh" - }, - { - "name": "Build Image", - "script": "build-image.sh" - }, - { - "name": "Generate Certificates", - "script": "generate-certificates.sh" - }, - { - "name": "Build Scripts", - "script": "build-scripts.sh" - }, - { - "name": "Create VMs", - "script": "create-vms.sh" - }, - { - "name": "Genesis", - "script": "genesis.sh" - }, - { - "name": "Join Masters", - "script": "join-masters.sh", - "arguments": [ - "n1" - ] - }, - { - "name": "Reprovision Genesis", - "script": "reprovision-genesis.sh", - "arguments": [ - "n1" - ] - }, - { - "name": "Hard Reboot Cluster", - "script": "hard-reboot-cluster.sh" - } - ], - "vm": { - "memory": 8096, - "names": [ - "n0", - "n1" - ], - "vcpus": 4 - } -} diff --git a/tools/g2/stages/build-scripts.sh b/tools/g2/stages/build-scripts.sh index f74ac6c0..c92a8312 100755 --- a/tools/g2/stages/build-scripts.sh +++ b/tools/g2/stages/build-scripts.sh @@ -18,3 +18,5 @@ docker run --rm -t \ --validators \ -o scripts \ config/*.yaml + +cat "${TEMP_DIR}"/config/*.yaml > "${TEMP_DIR}/nginx/promenade.yaml" diff --git a/tools/g2/stages/gate-setup.sh b/tools/g2/stages/gate-setup.sh index 211470e2..19981210 100755 --- a/tools/g2/stages/gate-setup.sh +++ b/tools/g2/stages/gate-setup.sh @@ -8,6 +8,10 @@ source "${GATE_UTILS}" registry_up registry_populate +# NginX for serving config files in the absence of Deckhand +nginx_down +nginx_up + # SSH setup ssh_setup_declare diff --git a/tools/g2/stages/join-masters.sh b/tools/g2/stages/join-masters.sh deleted file mode 100755 index e67d8043..00000000 --- a/tools/g2/stages/join-masters.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [ $# -le 0 ]; then - echo "Must specify at least one vm to join" - exit 1 -fi - -source "${GATE_UTILS}" - -JOIN_TARGETS="${*}" - -for NAME in ${JOIN_TARGETS}; do - rsync_cmd "${TEMP_DIR}"/scripts/*"${NAME}"* "${NAME}:/root/promenade/" - - ssh_cmd "${NAME}" "/root/promenade/join-${NAME}.sh" - ssh_cmd "${NAME}" "/root/promenade/validate-${NAME}.sh" - - # NOTE(mark-burnett): Ensure disk cache is flushed after join. - ssh_cmd "${NAME}" sync -done - -validate_cluster n0 - -validate_etcd_membership kubernetes n0 genesis "${*}" -validate_etcd_membership calico n0 n0 "${*}" diff --git a/tools/g2/stages/join-nodes.sh b/tools/g2/stages/join-nodes.sh new file mode 100755 index 00000000..ef454d64 --- /dev/null +++ b/tools/g2/stages/join-nodes.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -eu + +source "${GATE_UTILS}" + +declare -a ETCD_CLUSTERS +declare -a LABELS +declare -a NODES + +while getopts "e:l:n:v:" opt; do + case "${opt}" in + e) + ETCD_CLUSTERS+=("${OPTARG}") + ;; + l) + LABELS+=("${OPTARG}") + ;; + n) + NODES+=("${OPTARG}") + ;; + v) + VIA=${OPTARG} + ;; + *) + echo "Unknown option" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ $# -gt 0 ]; then + echo "Unknown arguments specified: ${*}" + exit 1 +fi + +SCRIPT_DIR="${TEMP_DIR}/curled-scripts" + +echo Etcd Clusters: "${ETCD_CLUSTERS[@]}" +echo Labels: "${LABELS[@]}" +echo Nodes: "${NODES[@]}" + +render_curl_url() { + NAME=${1} + shift + LABELS=(${@}) + + LABEL_PARAMS= + for label in "${LABELS[@]}"; do + LABEL_PARAMS+="&labels.dynamic=${label}" + done + + BASE_URL="http://promenade-api.ucp.svc.cluster.local/api/v1.0/join-scripts" + DESIGN_REF="design_ref=http://192.168.77.1:7777/promenade.yaml" + HOST_PARAMS="hostname=${NAME}&ip=$(config_vm_ip "${NAME}")" + + echo "'${BASE_URL}?${DESIGN_REF}&${HOST_PARAMS}${LABEL_PARAMS}'" +} + +mkdir -p "${SCRIPT_DIR}" + +for NAME in "${NODES[@]}"; do + log Building join script for node "${NAME}" + + ssh_cmd "${VIA}" curl "$(render_curl_url "${NAME}" "${LABELS[@]}")" > "${SCRIPT_DIR}/join-${NAME}.sh" + chmod 755 "${SCRIPT_DIR}/join-${NAME}.sh" + + log Joining node "${NAME}" + rsync_cmd "${SCRIPT_DIR}/join-${NAME}.sh" "${NAME}:/root/promenade/" + ssh_cmd "${NAME}" "/root/promenade/join-${NAME}.sh" +done + +for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do + IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}" + validate_etcd_membership "${etcd_validation_args[@]}" +done diff --git a/tools/g2/stages/teardown-nodes.sh b/tools/g2/stages/teardown-nodes.sh new file mode 100755 index 00000000..363d7925 --- /dev/null +++ b/tools/g2/stages/teardown-nodes.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -eu + +source "${GATE_UTILS}" + +declare -a ETCD_CLUSTERS +declare -a NODES + +RECREATE=0 + +while getopts "e:n:rv:" opt; do + case "${opt}" in + e) + ETCD_CLUSTERS+=("${OPTARG}") + ;; + n) + NODES+=("${OPTARG}") + ;; + r) + RECREATE=1 + ;; + v) + VIA=${OPTARG} + ;; + *) + echo "Unknown option" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ $# -gt 0 ]; then + echo "Unknown arguments specified: ${*}" + exit 1 +fi + +for NAME in "${NODES[@]}"; do + log Tearing down node "${NAME}" + promenade_teardown_node "${NAME}" "${VIA}" + vm_clean "${NAME}" + if [[ ${RECREATE} == "1" ]]; then + vm_create "${NAME}" + fi +done + +for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do + IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}" + validate_etcd_membership "${etcd_validation_args[@]}" +done diff --git a/tools/stop_gate.sh b/tools/stop_gate.sh index e4d299a9..b3ce00a2 100755 --- a/tools/stop_gate.sh +++ b/tools/stop_gate.sh @@ -14,3 +14,4 @@ source "${GATE_UTILS}" vm_clean_all net_clean registry_down +nginx_down