From 14e78330465c2b9d70548f9ad3dd0e5b315638e5 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 24 Jul 2017 10:41:51 -0500 Subject: [PATCH 01/10] Fix outdated documentation regarding proxies --- docs/configuration.md | 5 +++-- docs/getting-started.md | 19 ++----------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 606dc478..deba49c0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -85,7 +85,8 @@ The `Network` document must contain: - `dns_servers` - A list of upstream DNS server IPs. Optionally, proxy settings can be specified here as well. These should all -generally be set together: `http_proxy`, `https_proxy`, `no_proxy`. +generally be set together: `http_proxy`, `https_proxy`, `no_proxy`. `no_proxy` +must include all master IP addresses, and the `kubernetes` service name. Here's an example `Network` document: @@ -108,7 +109,7 @@ spec: - 8.8.4.4 http_proxy: http://proxy.example.com:8080 https_proxy: http://proxy.example.com:8080 - no_proxy: 192.168.77.10,127.0.0.1,kubernetes + no_proxy: 192.168.77.10,192.168.77.11,192.168.77.12,127.0.0.1,kubernetes,kubernetes.default.svc.cluster.local ``` The `Versions` document must define the Promenade image to be used and the diff --git a/docs/getting-started.md b/docs/getting-started.md index 667650da..0268c1da 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -39,13 +39,6 @@ Join the worker node: vagrant ssh n3 -c 'sudo /vagrant/configs/up.sh /vagrant/configs/n3.yaml' ``` -To use Promenade from behind a proxy, simply add proxy settings to the -promenade `Network` configuration document using the keys `http_proxy`, -`https_proxy`, and `no_proxy` before running `generate`. - -Note that it is important to specify `no_proxy` to include `kubernetes` and the -IP addresses of all the master nodes. - ### Building the image ```bash @@ -74,13 +67,5 @@ docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_pr ## Using Promenade Behind a Proxy -To use Promenade from behind a proxy, simply export `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY` environment variables on the vagrant host prior to executing the `genesis.sh` and `join.sh` scripts respectively. Alternatively, you may also export the `DOCKER_HTTP_PROXY`, `DOCKER_HTTPS_PROXY`, and `DOCKER_NO_PROXY` directly. Ensure you are running the script with `sudo -E` option to preserve the environment variables. - -```bash -vagrant ssh n0 -cd /vagrant -export DOCKER_HTTP_PROXY="http://proxy.server.com:8080" -export DOCKER_HTTPS_PROXY="https://proxy.server.com:8080" -export DOCKER_NO_PROXY="localhost,127.0.0.1" -sudo -E /vagrant/up.sh /vagrant/configs/n0.yaml -``` +To use Promenade from behind a proxy, use the proxy settings described in the +[configuration docs](configuration.md). From 527d08e5a232d3f5504a69a3e8ad23c0f381c8b5 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 24 Jul 2017 10:42:11 -0500 Subject: [PATCH 02/10] Call up.sh as an argument to bash in docs --- docs/getting-started.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 0268c1da..83205fde 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -23,20 +23,20 @@ vagrant up Start the genesis node: ```bash -vagrant ssh n0 -c 'sudo /vagrant/configs/up.sh /vagrant/configs/n0.yaml' +vagrant ssh n0 -c 'sudo bash /vagrant/configs/up.sh /vagrant/configs/n0.yaml' ``` Join the master nodes: ```bash -vagrant ssh n1 -c 'sudo /vagrant/configs/up.sh /vagrant/configs/n1.yaml' -vagrant ssh n2 -c 'sudo /vagrant/configs/up.sh /vagrant/configs/n2.yaml' +vagrant ssh n1 -c 'sudo bash /vagrant/configs/up.sh /vagrant/configs/n1.yaml' +vagrant ssh n2 -c 'sudo bash /vagrant/configs/up.sh /vagrant/configs/n2.yaml' ``` Join the worker node: ```bash -vagrant ssh n3 -c 'sudo /vagrant/configs/up.sh /vagrant/configs/n3.yaml' +vagrant ssh n3 -c 'sudo bash /vagrant/configs/up.sh /vagrant/configs/n3.yaml' ``` ### Building the image @@ -54,7 +54,7 @@ docker save -o promenade.tar promenade:local Then on a node: ```bash -PROMENADE_LOAD_IMAGE=/vagrant/promenade.tar /vagrant/up.sh /vagrant/path/to/node-config.yaml +PROMENADE_LOAD_IMAGE=/vagrant/promenade.tar bash /vagrant/up.sh /vagrant/path/to/node-config.yaml ``` To build the image from behind a proxy, you can: From 74bde0f6f3bfbb4457b02e6ea138fa014ccc40f9 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 24 Jul 2017 12:28:05 -0500 Subject: [PATCH 03/10] Add image build dev script --- docs/getting-started.md | 2 ++ tools/dev-build.sh | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100755 tools/dev-build.sh diff --git a/docs/getting-started.md b/docs/getting-started.md index 83205fde..5e3f7d89 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -57,6 +57,8 @@ Then on a node: PROMENADE_LOAD_IMAGE=/vagrant/promenade.tar bash /vagrant/up.sh /vagrant/path/to/node-config.yaml ``` +These commands are combined in a convenience script at `tools/dev-build.sh`. + To build the image from behind a proxy, you can: ```bash diff --git a/tools/dev-build.sh b/tools/dev-build.sh new file mode 100755 index 00000000..5852cbbe --- /dev/null +++ b/tools/dev-build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -ex + +echo === Cleaning up old data === +rm -rf promenade.tar configs +mkdir configs + +echo === Building image === +docker build -t quay.io/attcomdev/promenade:latest . + +echo === Generating updated configuration === +docker run --rm -t \ + -v $(pwd):/target quay.io/attcomdev/promenade:latest \ + promenade -v \ + generate \ + -c /target/example/vagrant-input-config.yaml \ + -o /target/configs + +echo === Saving image === +docker save -o promenade.tar quay.io/attcomdev/promenade:latest + +echo === Done === From d04f333686f7e062c6930cebe394b98a98a9c494 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 24 Jul 2017 12:37:39 -0500 Subject: [PATCH 04/10] Add skeleton for validation scripts * validate-bootstrap.sh is to be used to validate after genesis * validate-cluster.sh is to be used to validate a full cluster --- promenade/generator.py | 4 +- .../templates/generate/validate-bootstrap.sh | 5 + .../templates/generate/validate-cluster.sh | 6 + .../templates/include/common_validation.sh | 122 ++++++++++++++++++ 4 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 promenade/templates/generate/validate-bootstrap.sh create mode 100644 promenade/templates/generate/validate-cluster.sh create mode 100644 promenade/templates/include/common_validation.sh diff --git a/promenade/generator.py b/promenade/generator.py index 5ce3b19c..1792646f 100644 --- a/promenade/generator.py +++ b/promenade/generator.py @@ -30,13 +30,13 @@ class Generator: assert self.input_config['Cluster'].metadata['name'] \ == self.input_config['Network'].metadata['cluster'] - def generate_up_sh(self, output_dir): + def generate_additional_scripts(self, output_dir): r = renderer.Renderer(config=self.input_config, target_dir=output_dir) r.render_generate_files() def generate_all(self, output_dir): - self.generate_up_sh(output_dir) + self.generate_additional_scripts(output_dir) cluster = self.input_config['Cluster'] network = self.input_config['Network'] diff --git a/promenade/templates/generate/validate-bootstrap.sh b/promenade/templates/generate/validate-bootstrap.sh new file mode 100644 index 00000000..41cea1ca --- /dev/null +++ b/promenade/templates/generate/validate-bootstrap.sh @@ -0,0 +1,5 @@ +{% include "common_validation.sh" with context %} + +wait_for_ready_nodes 1 + +validate_kubectl_logs diff --git a/promenade/templates/generate/validate-cluster.sh b/promenade/templates/generate/validate-cluster.sh new file mode 100644 index 00000000..1d5dbd37 --- /dev/null +++ b/promenade/templates/generate/validate-cluster.sh @@ -0,0 +1,6 @@ +{% include "common_validation.sh" with context %} + +EXPECTED_NODE_COUNT={{ config['Cluster']['nodes'] | length }} +wait_for_ready_nodes $EXPECTED_NODE_COUNT + +validate_kubectl_logs diff --git a/promenade/templates/include/common_validation.sh b/promenade/templates/include/common_validation.sh new file mode 100644 index 00000000..cddee9db --- /dev/null +++ b/promenade/templates/include/common_validation.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +if [ "$(id -u)" != "0" ]; then + echo "This script must be run as root." 1>&2 + exit 1 +fi + +set -ex + +export KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml + +function log { + echo $* 1>&2 +} + +function report_kube_state { + log General cluster state report + kubectl get nodes 1>&2 + kubectl get --all-namespaces pods -o wide 1>&2 +} + +function fail { + report_kube_state + exit 1 +} + +function wait_for_ready_nodes { + set +x + + NODES=$1 + SECONDS=${2:-600} + + log $(date) Waiting $SECONDS seconds for $NODES Ready nodes. + + NODE_READY_JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[?(@.type=="Ready")]}{@.type}={@.status}{"\n"}{end}{end}' + + end=$(($(date +%s) + $SECONDS)) + while true; do + READY_NODE_COUNT=$(kubectl get nodes -o jsonpath="${NODE_READY_JSONPATH}" | grep "Ready=True" | wc -l) + if [ $NODES -ne $READY_NODE_COUNT ]; then + now=$(date +%s) + if [ $now -gt $end ]; then + log $(date) Nodes were not all ready before timeout. + kubectl get nodes 1>&2 + kubectl get --all-namespaces pods -o wide 1>&2 + exit 1 + fi + sleep 5 + else + log $(date) Found expected nodes. + break + fi + done + + set -x +} + +function wait_for_pod_termination { + set +x + + NAMESPACE=$1 + POD_NAME=$2 + SECONDS=${3:-120} + + log $(date) Waiting $SECONDS seconds for termination of pod $POD_NAME + + POD_PHASE_JSONPATH='{.status.phase}' + + end=$(($(date +%s) + $SECONDS)) + while true; do + POD_PHASE=$(kubectl --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME) + if [ "x$POD_PHASE" = "xSucceeded" ]; then + log $(date) Pod $POD_NAME succeeded. + break + elif [ "x$POD_PHASE" = "xFailed" ]; then + log $(date) Pod $POD_NAME failed. + kubectl --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 + fail + else + now=$(date +%s) + if [ $now -gt $end ]; then + log $(date) Pod did not terminate before timeout. + kubectl --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 + fail + fi + sleep 1 + fi + done + + set -x +} + +function validate_kubectl_logs { + NAMESPACE=default + POD_NAME=log-test-$(date +%s) + + cat <&2 + fail + fi +} From d3adf921aaa70e0e82bad222a58f17a393059c05 Mon Sep 17 00:00:00 2001 From: Stacey Fletcher Date: Wed, 26 Jul 2017 10:23:32 -0500 Subject: [PATCH 05/10] This PS adds a promenade yaml that templates the IPs and promenade image. The promenade yaml will be used by Jenkins and the IPs will be dynamic so we cannot have them hard-coded. The version of the Promenade image is also templated to allow for the Jenkins major.minor.patch tag to replace it. Creating a generated_configs directory that will be used to store the generated yaml and script files. --- tools/generated_configs/.gitignore | 0 tools/promenade-config.yaml | 90 ++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 tools/generated_configs/.gitignore create mode 100644 tools/promenade-config.yaml diff --git a/tools/generated_configs/.gitignore b/tools/generated_configs/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/tools/promenade-config.yaml b/tools/promenade-config.yaml new file mode 100644 index 00000000..bb3afd90 --- /dev/null +++ b/tools/promenade-config.yaml @@ -0,0 +1,90 @@ +--- +apiVersion: promenade/v1 +kind: Cluster +metadata: + name: example + target: none +spec: + nodes: + n0: + ip: GENESIS + kubernetes_interface: G_IFACE + roles: + - master + - genesis + additional_labels: + - beta.kubernetes.io/arch=amd64 + n1: + ip: MASTER_1 + kubernetes_interface: M1_IFACE + roles: + - master + additional_labels: + - beta.kubernetes.io/arch=amd64 + n2: + ip: MASTER_2 + kubernetes_interface: M2_IFACE + roles: + - master + additional_labels: + - beta.kubernetes.io/arch=amd64 + n3: + ip: WORKER + kubernetes_interface: W_IFACE + roles: + - worker + additional_labels: + - beta.kubernetes.io/arch=amd64 +--- +apiVersion: promenade/v1 +kind: Network +metadata: + cluster: example + name: example + target: all +spec: + cluster_domain: cluster.local + cluster_dns: 10.96.0.10 + kube_service_ip: 10.96.0.1 + pod_ip_cidr: 10.97.0.0/16 + service_ip_cidr: 10.96.0.0/16 + calico_etcd_service_ip: 10.96.232.136 + dns_servers: + - 8.8.8.8 + - 8.8.4.4 + #http_proxy: http://proxy.example.com:8080 + #https_proxy: https://proxy.example.com:8080 +--- +apiVersion: promenade/v1 +kind: Versions +metadata: + cluster: example + name: example + target: all +spec: + images: + armada: quay.io/attcomdev/armada:latest + calico: + cni: quay.io/calico/cni:v1.9.1 + etcd: quay.io/coreos/etcd:v3.2.1 + node: quay.io/calico/node:v1.3.0 + policy-controller: quay.io/calico/kube-policy-controller:v0.6.0 + kubernetes: + apiserver: gcr.io/google_containers/hyperkube-amd64:v1.6.4 + controller-manager: quay.io/attcomdev/kube-controller-manager:v1.6.4 + dns: + dnsmasq: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.2 + kubedns: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.2 + sidecar: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.2 + etcd: quay.io/coreos/etcd:v3.2.1 + kubectl: gcr.io/google_containers/hyperkube-amd64:v1.6.4 + proxy: gcr.io/google_containers/hyperkube-amd64:v1.6.4 + scheduler: gcr.io/google_containers/hyperkube-amd64:v1.6.4 + promenade: quay.io/attcomdev/promenade:latest + tiller: gcr.io/kubernetes-helm/tiller:v2.4.2 + packages: + docker: docker.io=1.12.6-0ubuntu1~16.04.1 + dnsmasq: dnsmasq=2.75-1ubuntu0.16.04.2 + socat: socat=1.7.3.1-1 + additional_packages: + - ceph-common=10.2.7-0ubuntu0.16.04.1 From a15c7cc1fc9fc0fb4c1473171b05f9059b710ce2 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Wed, 26 Jul 2017 15:47:33 -0500 Subject: [PATCH 06/10] Freeze armada version for testing --- tools/promenade-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/promenade-config.yaml b/tools/promenade-config.yaml index bb3afd90..7b922e22 100644 --- a/tools/promenade-config.yaml +++ b/tools/promenade-config.yaml @@ -63,7 +63,7 @@ metadata: target: all spec: images: - armada: quay.io/attcomdev/armada:latest + armada: quay.io/attcomdev/armada:v0.5.1 calico: cni: quay.io/calico/cni:v1.9.1 etcd: quay.io/coreos/etcd:v3.2.1 From a54d2bedeb3d125bf7ae2d30b2111a0005c9c6ad Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Fri, 28 Jul 2017 10:18:31 -0500 Subject: [PATCH 07/10] Report additional docker state --- promenade/templates/include/common_validation.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/promenade/templates/include/common_validation.sh b/promenade/templates/include/common_validation.sh index cddee9db..459a2016 100644 --- a/promenade/templates/include/common_validation.sh +++ b/promenade/templates/include/common_validation.sh @@ -13,6 +13,21 @@ function log { echo $* 1>&2 } +function report_docker_exited_containers { + for container_id in $(docker ps -q --filter "status=exited"); do + log Report for exited container $container_id + docker inspect $container_id + docker logs $container_id + done +} + +function report_docker_state { + log General docker state report + docker info + docker ps -a + report_docker_exited_containers +} + function report_kube_state { log General cluster state report kubectl get nodes 1>&2 @@ -20,6 +35,7 @@ function report_kube_state { } function fail { + report_docker_state report_kube_state exit 1 } From 3b3971420af2011d8d533b4fc6edcbce85b59df0 Mon Sep 17 00:00:00 2001 From: Stacey Fletcher Date: Fri, 28 Jul 2017 10:30:06 -0500 Subject: [PATCH 08/10] Update configuration file to replace HostName. This change will allow for Jenkins to replace the hostname with the newly provisioned VM's hostname. --- tools/promenade-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/promenade-config.yaml b/tools/promenade-config.yaml index 7b922e22..3e4f5a1e 100644 --- a/tools/promenade-config.yaml +++ b/tools/promenade-config.yaml @@ -6,7 +6,7 @@ metadata: target: none spec: nodes: - n0: + G_HOSTNAME: ip: GENESIS kubernetes_interface: G_IFACE roles: @@ -14,21 +14,21 @@ spec: - genesis additional_labels: - beta.kubernetes.io/arch=amd64 - n1: + M1_HOSTNAME: ip: MASTER_1 kubernetes_interface: M1_IFACE roles: - master additional_labels: - beta.kubernetes.io/arch=amd64 - n2: + M2_HOSTNAME: ip: MASTER_2 kubernetes_interface: M2_IFACE roles: - master additional_labels: - beta.kubernetes.io/arch=amd64 - n3: + W_HOSTNAME: ip: WORKER kubernetes_interface: W_IFACE roles: From 7f34a8e1f9d1ba4743878dc756c286dd903e9477 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Fri, 28 Jul 2017 11:23:13 -0500 Subject: [PATCH 09/10] Add missed call to common fail path --- promenade/templates/include/common_validation.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/promenade/templates/include/common_validation.sh b/promenade/templates/include/common_validation.sh index 459a2016..797b9907 100644 --- a/promenade/templates/include/common_validation.sh +++ b/promenade/templates/include/common_validation.sh @@ -57,9 +57,7 @@ function wait_for_ready_nodes { now=$(date +%s) if [ $now -gt $end ]; then log $(date) Nodes were not all ready before timeout. - kubectl get nodes 1>&2 - kubectl get --all-namespaces pods -o wide 1>&2 - exit 1 + fail fi sleep 5 else From 5e10d3a3415109a8ab52c622f07be3d6cfc2dd5e Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Fri, 28 Jul 2017 13:58:14 -0500 Subject: [PATCH 10/10] Overwrite /etc/resolv.conf --- promenade/templates/common/usr/local/bin/bootstrap | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/promenade/templates/common/usr/local/bin/bootstrap b/promenade/templates/common/usr/local/bin/bootstrap index 6183c443..fe65758c 100755 --- a/promenade/templates/common/usr/local/bin/bootstrap +++ b/promenade/templates/common/usr/local/bin/bootstrap @@ -14,3 +14,14 @@ apt-get install -y --no-install-recommends \ systemctl daemon-reload systemctl enable kubelet systemctl restart kubelet + + +cat < /etc/resolv.conf +options timeout:1 attempts:1 + +nameserver 127.0.0.1 + +{%- for server in config['Network']['dns_servers'] %} +nameserver {{ server }} +{%- endfor %} +EOF