Add join-scripts endpoint

* Adds initial join-scripts API
* Updates resiliency test to leverage API for joining

Change-Id: Ibe0d42b8f4f4a3e1f6f102dee85a22cb8f78f8ec
This commit is contained in:
Mark Burnett 2017-11-02 14:37:14 -05:00
parent 6caf7fb54d
commit b4d9596468
20 changed files with 361 additions and 159 deletions

View File

@ -38,7 +38,7 @@ static.labels
Used to set configuration options in the generated script
Responses
- 204 No Content: Scripts generated successfully
- 200 OK: Script returned as response body
- 400 Bad Request: One or more query parameters is missing or misspelled

View File

@ -80,6 +80,16 @@ class Builder:
def build_node(self, node_document, *, output_dir):
node_name = node_document['metadata']['name']
LOG.info('Building script for node %s', node_name)
script = self.build_node_script(node_name)
_write_script(output_dir, _join_name(node_name), script)
if self.validators:
validate_script = self._build_node_validate_script(node_name)
_write_script(output_dir, 'validate-%s.sh' % node_name,
validate_script)
def build_node_script(self, node_name):
sub_config = self.config.extract_node_config(node_name)
file_spec_paths = [
f['path'] for f in self.config.get_path('HostSystem:files', [])
@ -88,20 +98,17 @@ class Builder:
tarball = renderer.build_tarball_from_roles(
config=sub_config, roles=['common', 'join'], file_specs=file_specs)
script = renderer.render_template(
return renderer.render_template(
sub_config,
template='scripts/join.sh',
context={
'tarball': tarball
})
_write_script(output_dir, _join_name(node_name), script)
if self.validators:
validate_script = renderer.render_template(
sub_config, template='scripts/validate-join.sh')
_write_script(output_dir, 'validate-%s.sh' % node_name,
validate_script)
def _build_node_validate_script(self, node_name):
sub_config = self.config.extract_node_config(node_name)
return renderer.render_template(
sub_config, template='scripts/validate-join.sh')
def _fetch_tar_content(*, url, path):

View File

@ -2,6 +2,7 @@ from . import exceptions, logging, validation
import copy
import jinja2
import jsonpath_ng
import requests
import yaml
__all__ = ['Configuration']
@ -32,6 +33,16 @@ class Configuration:
return cls(documents=documents, **kwargs)
@classmethod
def from_design_ref(cls, design_ref):
response = requests.get(design_ref)
response.raise_for_status()
documents = list(yaml.safe_load_all(response.text))
validation.check_schemas(documents)
return cls(documents=documents)
def __getitem__(self, path):
value = self.get_path(path)
if value:
@ -125,6 +136,10 @@ class Configuration:
return data
return default
def append(self, item):
validation.check_schema(item)
self.documents.append(item)
def _matches_filter(document, *, schema, labels):
matches = True

View File

@ -16,6 +16,7 @@ import falcon
from promenade.control.base import BaseResource, PromenadeRequest
from promenade.control.health_api import HealthResource
from promenade.control.join_scripts import JoinScriptsResource
from promenade.control.middleware import (AuthMiddleware, ContextMiddleware,
LoggingMiddleware)
from promenade import exceptions as exc
@ -37,6 +38,7 @@ def start_api():
v1_0_routes = [
# API for managing region data
('/health', HealthResource()),
('/join-scripts', JoinScriptsResource()),
]
# Set up the 1.0 routes

View File

@ -0,0 +1,95 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from promenade.control.base import BaseResource
from promenade.builder import Builder
from promenade.config import Configuration
from promenade import logging
import falcon
import kubernetes
import random
LOG = logging.getLogger(__name__)
class JoinScriptsResource(BaseResource):
"""
Lists the versions supported by this API
"""
def on_get(self, req, resp):
design_ref = req.get_param('design_ref', required=True)
ip = req.get_param('ip', required=True)
hostname = req.get_param('hostname', required=True)
dynamic_labels = _get_param_list(req, 'labels.dynamic')
static_labels = _get_param_list(req, 'labels.static')
join_ip = _get_join_ip()
config = Configuration.from_design_ref(design_ref)
node_document = {
'schema': 'promenade/KubernetesNode/v1',
'metadata': {
'name': hostname,
'schema': 'metadata/Document/v1',
'layeringDefinition': {
'abstract': False,
'layer': 'site'
},
},
'data': {
'hostname': hostname,
'ip': ip,
'join_ip': join_ip,
'labels': {
'dynamic': dynamic_labels,
'static': static_labels,
},
},
}
config.append(node_document)
builder = Builder(config)
script = builder.build_node_script(hostname)
resp.body = script
resp.content_type = 'text/x-shellscript'
resp.status = falcon.HTTP_200
def _get_join_ip():
# TODO(mark-burnett): Handle errors
kubernetes.config.load_incluster_config()
client = kubernetes.client.CoreV1Api()
response = client.list_node(label_selector='kubernetes-apiserver=enabled')
# Ignore bandit false positive: B311:blacklist
# The choice of which master to join to is a load-balancing concern, not a
# security concern.
return random.choice(list(map(_extract_ip, response.items))) # nosec
def _extract_ip(item):
for address in item.status.addresses:
if address.type == 'InternalIP':
return address.address
def _get_param_list(req, name):
values = req.get_param_as_list(name)
if values:
return values
else:
return []

View File

@ -33,6 +33,8 @@ def insert_charts_into_bundler(bundler):
for root, _dirnames, filenames in os.walk(
'/opt/promenade/charts', followlinks=True):
for source_filename in filenames:
if _source_file_is_excluded(source_filename):
continue
source_path = os.path.join(root, source_filename)
destination_path = os.path.join('etc/genesis/armada/assets/charts',
os.path.relpath(
@ -139,6 +141,10 @@ def _default_no_proxy(network_config):
return ','.join(include)
def _source_file_is_excluded(source_filename):
return source_filename.endswith('.tgz')
def _yaml_safe_dump_all(documents):
f = io.StringIO()
yaml.safe_dump_all(documents, f)

View File

@ -4,7 +4,8 @@ jinja2==2.9.6
jsonpath-ng==1.4.3
jsonschema==2.6.0
keystonemiddleware==4.17.0
oslo.context==2.14.0
kubernetes==3.0.0
oslo.context==2.19.2
pastedeploy==1.5.2
pbr==3.0.1
pyyaml==3.12

View File

@ -1,24 +1,28 @@
Babel==2.5.1
cachetools==2.0.1
certifi==2017.7.27.1
chardet==3.0.4
click==6.7
debtcollector==1.18.0
decorator==4.1.2
falcon==1.2.0
google-auth==1.2.0
idna==2.6
ipaddress==1.0.18
iso8601==0.1.12
Jinja2==2.9.6
jsonpath-ng==1.4.3
jsonschema==2.6.0
keystoneauth1==3.2.0
keystonemiddleware==4.17.0
kubernetes==3.0.0
MarkupSafe==1.0
monotonic==1.4
msgpack-python==0.4.8
netaddr==0.7.19
netifaces==0.10.6
oslo.config==5.0.0
oslo.context==2.14.0
oslo.context==2.19.2
oslo.i18n==3.18.0
oslo.log==3.32.0
oslo.serialization==2.21.2
@ -27,6 +31,8 @@ PasteDeploy==1.5.2
pbr==3.0.1
ply==3.10
positional==1.2.1
pyasn1==0.3.7
pyasn1-modules==0.1.5
pycadf==2.6.0
pyinotify==0.9.6
pyparsing==2.2.0
@ -37,9 +43,11 @@ pytz==2017.3
PyYAML==3.12
requests==2.18.4
rfc3986==1.1.0
rsa==3.4.2
six==1.11.0
stevedore==1.27.1
urllib3==1.22
uWSGI==2.0.15
WebOb==1.7.3
websocket-client==0.40.0
wrapt==1.10.11

View File

@ -8,6 +8,7 @@ source "$LIB_DIR"/const.sh
source "$LIB_DIR"/etcd.sh
source "$LIB_DIR"/kube.sh
source "$LIB_DIR"/log.sh
source "$LIB_DIR"/nginx.sh
source "$LIB_DIR"/promenade.sh
source "$LIB_DIR"/registry.sh
source "$LIB_DIR"/ssh.sh

18
tools/g2/lib/nginx.sh Normal file
View File

@ -0,0 +1,18 @@
nginx_down() {
REGISTRY_ID=$(docker ps -qa -f name=promenade-nginx)
if [ "x${REGISTRY_ID}" != "x" ]; then
log Removing nginx server
docker rm -fv "${REGISTRY_ID}" &>> "${LOG_FILE}"
fi
}
nginx_up() {
log Starting nginx server to serve configuration files
mkdir -p "${TEMP_DIR}/nginx"
docker run -d \
-p 7777:80 \
--restart=always \
--name promenade-nginx \
-v "${TEMP_DIR}/nginx:/usr/share/nginx/html:ro" \
nginx:stable &>> "${LOG_FILE}"
}

View File

@ -29,18 +29,42 @@
},
{
"name": "Join Masters",
"script": "join-masters.sh",
"script": "join-nodes.sh",
"arguments": [
"n1",
"n2",
"n3"
"-v", "n0",
"-n", "n1",
"-n", "n2",
"-n", "n3",
"-l", "calico-etcd=enabled",
"-l", "kubernetes-apiserver=enabled",
"-l", "kubernetes-controller-manager=enabled",
"-l", "kubernetes-etcd=enabled",
"-l", "kubernetes-scheduler=enabled",
"-l", "ucp-control-plane=enabled",
"-e", "kubernetes n0 genesis n1 n2 n3",
"-e", "calico n0 n0 n1 n2 n3"
]
},
{
"name": "Reprovision Genesis",
"script": "reprovision-genesis.sh",
"name": "Teardown Genesis",
"script": "teardown-nodes.sh",
"arguments": [
"n1 n2 n3"
"-v", "n1",
"-n", "n0",
"-r",
"-e", "kubernetes n1 n1 n2 n3",
"-e", "calico n1 n1 n2 n3"
]
},
{
"name": "Join n0 as Worker",
"script": "join-nodes.sh",
"arguments": [
"-v", "n1",
"-n", "n0",
"-l", "ucp-control-plane=enabled",
"-e", "kubernetes n1 n1 n2 n3",
"-e", "calico n1 n1 n2 n3"
]
},
{

View File

@ -29,18 +29,42 @@
},
{
"name": "Join Masters",
"script": "join-masters.sh",
"script": "join-nodes.sh",
"arguments": [
"n1",
"n2",
"n3"
"-v", "n0",
"-n", "n1",
"-n", "n2",
"-n", "n3",
"-l", "calico-etcd=enabled",
"-l", "kubernetes-apiserver=enabled",
"-l", "kubernetes-controller-manager=enabled",
"-l", "kubernetes-etcd=enabled",
"-l", "kubernetes-scheduler=enabled",
"-l", "ucp-control-plane=enabled",
"-e", "kubernetes n0 genesis n1 n2 n3",
"-e", "calico n0 n0 n1 n2 n3"
]
},
{
"name": "Reprovision Genesis",
"script": "reprovision-genesis.sh",
"name": "Teardown Genesis",
"script": "teardown-nodes.sh",
"arguments": [
"n1 n2 n3"
"-v", "n1",
"-n", "n0",
"-r",
"-e", "kubernetes n1 n1 n2 n3",
"-e", "calico n1 n1 n2 n3"
]
},
{
"name": "Join n0 as Worker",
"script": "join-nodes.sh",
"arguments": [
"-v", "n1",
"-n", "n0",
"-l", "ucp-control-plane=enabled",
"-e", "kubernetes n1 n1 n2 n3",
"-e", "calico n1 n1 n2 n3"
]
},
{

View File

@ -1,50 +0,0 @@
{
"configuration": [
"examples/basic"
],
"stages": [
{
"name": "Gate Setup",
"script": "gate-setup.sh"
},
{
"name": "Build Image",
"script": "build-image.sh"
},
{
"name": "Generate Certificates",
"script": "generate-certificates.sh"
},
{
"name": "Build Scripts",
"script": "build-scripts.sh"
},
{
"name": "Create VMs",
"script": "create-vms.sh"
},
{
"name": "Genesis",
"script": "genesis.sh"
},
{
"name": "Join Masters",
"script": "join-masters.sh",
"arguments": [
"n1"
]
},
{
"name": "Hard Reboot Cluster",
"script": "hard-reboot-cluster.sh"
}
],
"vm": {
"memory": 2048,
"names": [
"n0",
"n1"
],
"vcpus": 2
}
}

View File

@ -1,57 +0,0 @@
{
"configuration": [
"examples/complete"
],
"stages": [
{
"name": "Gate Setup",
"script": "gate-setup.sh"
},
{
"name": "Build Image",
"script": "build-image.sh"
},
{
"name": "Generate Certificates",
"script": "generate-certificates.sh"
},
{
"name": "Build Scripts",
"script": "build-scripts.sh"
},
{
"name": "Create VMs",
"script": "create-vms.sh"
},
{
"name": "Genesis",
"script": "genesis.sh"
},
{
"name": "Join Masters",
"script": "join-masters.sh",
"arguments": [
"n1"
]
},
{
"name": "Reprovision Genesis",
"script": "reprovision-genesis.sh",
"arguments": [
"n1"
]
},
{
"name": "Hard Reboot Cluster",
"script": "hard-reboot-cluster.sh"
}
],
"vm": {
"memory": 8096,
"names": [
"n0",
"n1"
],
"vcpus": 4
}
}

View File

@ -18,3 +18,5 @@ docker run --rm -t \
--validators \
-o scripts \
config/*.yaml
cat "${TEMP_DIR}"/config/*.yaml > "${TEMP_DIR}/nginx/promenade.yaml"

View File

@ -8,6 +8,10 @@ source "${GATE_UTILS}"
registry_up
registry_populate
# NginX for serving config files in the absence of Deckhand
nginx_down
nginx_up
# SSH setup
ssh_setup_declare

View File

@ -1,27 +0,0 @@
#!/usr/bin/env bash
set -e
if [ $# -le 0 ]; then
echo "Must specify at least one vm to join"
exit 1
fi
source "${GATE_UTILS}"
JOIN_TARGETS="${*}"
for NAME in ${JOIN_TARGETS}; do
rsync_cmd "${TEMP_DIR}"/scripts/*"${NAME}"* "${NAME}:/root/promenade/"
ssh_cmd "${NAME}" "/root/promenade/join-${NAME}.sh"
ssh_cmd "${NAME}" "/root/promenade/validate-${NAME}.sh"
# NOTE(mark-burnett): Ensure disk cache is flushed after join.
ssh_cmd "${NAME}" sync
done
validate_cluster n0
validate_etcd_membership kubernetes n0 genesis "${*}"
validate_etcd_membership calico n0 n0 "${*}"

77
tools/g2/stages/join-nodes.sh Executable file
View File

@ -0,0 +1,77 @@
#!/usr/bin/env bash
set -eu
source "${GATE_UTILS}"
declare -a ETCD_CLUSTERS
declare -a LABELS
declare -a NODES
while getopts "e:l:n:v:" opt; do
case "${opt}" in
e)
ETCD_CLUSTERS+=("${OPTARG}")
;;
l)
LABELS+=("${OPTARG}")
;;
n)
NODES+=("${OPTARG}")
;;
v)
VIA=${OPTARG}
;;
*)
echo "Unknown option"
exit 1
;;
esac
done
shift $((OPTIND-1))
if [ $# -gt 0 ]; then
echo "Unknown arguments specified: ${*}"
exit 1
fi
SCRIPT_DIR="${TEMP_DIR}/curled-scripts"
echo Etcd Clusters: "${ETCD_CLUSTERS[@]}"
echo Labels: "${LABELS[@]}"
echo Nodes: "${NODES[@]}"
render_curl_url() {
NAME=${1}
shift
LABELS=(${@})
LABEL_PARAMS=
for label in "${LABELS[@]}"; do
LABEL_PARAMS+="&labels.dynamic=${label}"
done
BASE_URL="http://promenade-api.ucp.svc.cluster.local/api/v1.0/join-scripts"
DESIGN_REF="design_ref=http://192.168.77.1:7777/promenade.yaml"
HOST_PARAMS="hostname=${NAME}&ip=$(config_vm_ip "${NAME}")"
echo "'${BASE_URL}?${DESIGN_REF}&${HOST_PARAMS}${LABEL_PARAMS}'"
}
mkdir -p "${SCRIPT_DIR}"
for NAME in "${NODES[@]}"; do
log Building join script for node "${NAME}"
ssh_cmd "${VIA}" curl "$(render_curl_url "${NAME}" "${LABELS[@]}")" > "${SCRIPT_DIR}/join-${NAME}.sh"
chmod 755 "${SCRIPT_DIR}/join-${NAME}.sh"
log Joining node "${NAME}"
rsync_cmd "${SCRIPT_DIR}/join-${NAME}.sh" "${NAME}:/root/promenade/"
ssh_cmd "${NAME}" "/root/promenade/join-${NAME}.sh"
done
for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do
IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}"
validate_etcd_membership "${etcd_validation_args[@]}"
done

View File

@ -0,0 +1,51 @@
#!/usr/bin/env bash
set -eu
source "${GATE_UTILS}"
declare -a ETCD_CLUSTERS
declare -a NODES
RECREATE=0
while getopts "e:n:rv:" opt; do
case "${opt}" in
e)
ETCD_CLUSTERS+=("${OPTARG}")
;;
n)
NODES+=("${OPTARG}")
;;
r)
RECREATE=1
;;
v)
VIA=${OPTARG}
;;
*)
echo "Unknown option"
exit 1
;;
esac
done
shift $((OPTIND-1))
if [ $# -gt 0 ]; then
echo "Unknown arguments specified: ${*}"
exit 1
fi
for NAME in "${NODES[@]}"; do
log Tearing down node "${NAME}"
promenade_teardown_node "${NAME}" "${VIA}"
vm_clean "${NAME}"
if [[ ${RECREATE} == "1" ]]; then
vm_create "${NAME}"
fi
done
for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do
IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}"
validate_etcd_membership "${etcd_validation_args[@]}"
done

View File

@ -14,3 +14,4 @@ source "${GATE_UTILS}"
vm_clean_all
net_clean
registry_down
nginx_down