diff --git a/.gitignore b/.gitignore index 3b828e16..5b4b3a6b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__ /*.log /.python-version /build +/conformance /promenade.egg-info /tmp .tox/ diff --git a/charts/coredns/templates/bin/_anchor.tpl b/charts/coredns/templates/bin/_anchor.tpl index 83450669..c8187967 100644 --- a/charts/coredns/templates/bin/_anchor.tpl +++ b/charts/coredns/templates/bin/_anchor.tpl @@ -24,21 +24,22 @@ promenade { log stdout } -{{ .Values.coredns.cluster_domain }} { - kubernetes { +. { + kubernetes{{- range .Values.coredns.kubernetes_zones }} {{ . -}}{{- end }} { endpoint https://{{ .Values.network.kubernetes_netloc }} tls /etc/coredns/coredns.pem /etc/coredns/coredns-key.pem /etc/coredns/cluster-ca.pem + + pods insecure } + {{- if .Values.coredns.upstream_nameservers }} + {{ range .Values.coredns.upstream_nameservers }} + proxy . {{ . }} + {{- end }} + {{- end }} + loadbalance cache {{ .Values.coredns.cache.ttl }} - errors stdout - log stdout -} -. { - {{- if .Values.coredns.upstream_nameservers }} - proxy . {{- range .Values.coredns.upstream_nameservers }} {{ . -}}{{- end }} - {{- end }} errors stdout log stdout } diff --git a/charts/coredns/values.yaml b/charts/coredns/values.yaml index 56406d00..f777a72e 100644 --- a/charts/coredns/values.yaml +++ b/charts/coredns/values.yaml @@ -10,7 +10,8 @@ tls: key: placeholder coredns: - cluster_domain: cluster.local + kubernetes_zones: + - cluster.local cache: ttl: 60 host_etc_path: /etc/coredns @@ -28,7 +29,7 @@ coredns: images: anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 - coredns: coredns/coredns:011 + coredns: coredns/coredns:0.9.9 network: kubernetes_netloc: 10.96.0.1 diff --git a/examples/basic/KubernetesNetwork.yaml b/examples/basic/KubernetesNetwork.yaml index 0b654a37..b5755010 100644 --- a/examples/basic/KubernetesNetwork.yaml +++ b/examples/basic/KubernetesNetwork.yaml @@ -12,6 +12,7 @@ data: service_ip: 10.96.0.10 bootstrap_validation_checks: - calico-etcd.kube-system.svc.cluster.local + - google.com - kubernetes-etcd.kube-system.svc.cluster.local - kubernetes.default.svc.cluster.local upstream_servers: diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index 7731bd29..2b11c199 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -471,7 +471,10 @@ data: no_hooks: true values: coredns: - cluster_domain: cluster.local + kubernetes_zones: + - cluster.local + - 10.96.0.0/16 + - 10.97.0.0/16 upstream_nameservers: - 8.8.8.8 - 8.8.4.4 @@ -493,7 +496,7 @@ data: namespace: kube-system images: anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 - coredns: coredns/coredns:011 + coredns: coredns/coredns:0.9.9 tls: ca: placeholder cert: placeholder diff --git a/examples/complete/KubernetesNetwork.yaml b/examples/complete/KubernetesNetwork.yaml index 0b654a37..b5755010 100644 --- a/examples/complete/KubernetesNetwork.yaml +++ b/examples/complete/KubernetesNetwork.yaml @@ -12,6 +12,7 @@ data: service_ip: 10.96.0.10 bootstrap_validation_checks: - calico-etcd.kube-system.svc.cluster.local + - google.com - kubernetes-etcd.kube-system.svc.cluster.local - kubernetes.default.svc.cluster.local upstream_servers: diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml index 13de1810..90d56e2a 100644 --- a/examples/complete/armada-resources.yaml +++ b/examples/complete/armada-resources.yaml @@ -500,7 +500,10 @@ data: no_hooks: true values: coredns: - cluster_domain: cluster.local + kubernetes_zones: + - cluster.local + - 10.96.0.0/16 + - 10.97.0.0/16 upstream_nameservers: - 8.8.8.8 - 8.8.4.4 @@ -522,7 +525,7 @@ data: namespace: kube-system images: anchor: gcr.io/google_containers/hyperkube-amd64:v1.8.0 - coredns: coredns/coredns:011 + coredns: coredns/coredns:0.9.9 tls: ca: placeholder cert: placeholder diff --git a/tools/g2/lib/kube.sh b/tools/g2/lib/kube.sh index 49218417..20ff6d2a 100644 --- a/tools/g2/lib/kube.sh +++ b/tools/g2/lib/kube.sh @@ -1,3 +1,9 @@ +kubectl_apply() { + VIA=${1} + FILE=${2} + ssh_cmd ${VIA} "cat ${FILE} | kubectl apply -f -" +} + kubectl_cmd() { VIA=${1} @@ -5,3 +11,34 @@ kubectl_cmd() { ssh_cmd ${VIA} kubectl ${@} } + +kubectl_wait_for_pod() { + VIA=${1} + NAMESPACE=${2} + POD_NAME=${3} + SEC=${4:-600} + log Waiting ${SEC} seconds for termination of pod ${POD_NAME} + + POD_PHASE_JSONPATH='{.status.phase}' + + end=$(($(date +%s) + $SEC)) + while true; do + POD_PHASE=$(kubectl_cmd ${VIA} --request-timeout 10s --namespace ${NAMESPACE} get -o jsonpath="${POD_PHASE_JSONPATH}" pod ${POD_NAME}) + if [[ ${POD_PHASE} = "Succeeded" ]]; then + log Pod ${POD_NAME} succeeded. + break + elif [[ $POD_PHASE = "Failed" ]]; then + log Pod ${POD_NAME} failed. + kubectl_cmd ${VIA} --request-timeout 10s --namespace ${NAMESPACE} get -o yaml pod ${POD_NAME} 1>&2 + fail + else + now=$(date +%s) + if [ $now -gt $end ]; then + log Pod did not terminate before timeout. + kubectl_cmd ${VIA} --request-timeout 10s --namespace ${NAMESPACE} get -o yaml pod ${POD_NAME} 1>&2 + fail + fi + sleep 1 + fi + done +} diff --git a/tools/g2/manifests/conformance.json b/tools/g2/manifests/conformance.json new file mode 100644 index 00000000..f90e00a4 --- /dev/null +++ b/tools/g2/manifests/conformance.json @@ -0,0 +1,61 @@ +{ + "configuration": [ + "examples/basic" + ], + "stages": [ + { + "name": "Gate Setup", + "script": "gate-setup.sh" + }, + { + "name": "Build Image", + "script": "build-image.sh" + }, + { + "name": "Generate Certificates", + "script": "generate-certificates.sh" + }, + { + "name": "Build Scripts", + "script": "build-scripts.sh" + }, + { + "name": "Create VMs", + "script": "create-vms.sh" + }, + { + "name": "Genesis", + "script": "genesis.sh" + }, + { + "name": "Join Masters", + "script": "join-masters.sh", + "arguments": [ + "n1", + "n2", + "n3" + ] + }, + { + "name": "Reprovision Genesis", + "script": "reprovision-genesis.sh", + "arguments": [ + "n1 n2 n3" + ] + }, + { + "name": "Check Conformance", + "script": "conformance.sh" + } + ], + "vm": { + "memory": 2048, + "names": [ + "n0", + "n1", + "n2", + "n3" + ], + "vcpus": 2 + } +} diff --git a/tools/g2/sonobuoy.yaml b/tools/g2/sonobuoy.yaml new file mode 100644 index 00000000..56bbd062 --- /dev/null +++ b/tools/g2/sonobuoy.yaml @@ -0,0 +1,302 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: heptio-sonobuoy +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + component: sonobuoy + name: sonobuoy-serviceaccount + namespace: heptio-sonobuoy +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + labels: + component: sonobuoy + name: sonobuoy-serviceaccount +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sonobuoy-serviceaccount +subjects: +- kind: ServiceAccount + name: sonobuoy-serviceaccount + namespace: heptio-sonobuoy +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + component: sonobuoy + name: sonobuoy-serviceaccount + namespace: heptio-sonobuoy +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +--- +apiVersion: v1 +data: + config.json: | + { + "Description": "EXAMPLE", + "Filters": { + "LabelSelector": "", + "Namespaces": ".*" + }, + "PluginNamespace": "heptio-sonobuoy", + "Plugins": [ + { + "name": "e2e" + } + ], + "Resources": [ + "CertificateSigningRequests", + "ClusterRoleBindings", + "ClusterRoles", + "ComponentStatuses", + "CustomResourceDefinitions", + "Nodes", + "PersistentVolumes", + "PodSecurityPolicies", + "ServerVersion", + "StorageClasses", + "ConfigMaps", + "DaemonSets", + "Deployments", + "Endpoints", + "Events", + "HorizontalPodAutoscalers", + "Ingresses", + "Jobs", + "LimitRanges", + "PersistentVolumeClaims", + "Pods", + "PodDisruptionBudgets", + "PodTemplates", + "ReplicaSets", + "ReplicationControllers", + "ResourceQuotas", + "RoleBindings", + "Roles", + "ServerGroups", + "ServiceAccounts", + "Services", + "StatefulSets" + ], + "ResultsDir": "/tmp/sonobuoy", + "Server": { + "advertiseaddress": "sonobuoy-master:8080", + "bindaddress": "0.0.0.0", + "bindport": 8080, + "timeoutseconds": 5400 + }, + "Version": "v0.9.0" + } +kind: ConfigMap +metadata: + labels: + component: sonobuoy + name: sonobuoy-config-cm + namespace: heptio-sonobuoy +--- +apiVersion: v1 +data: + e2e.yaml: | + driver: Job + name: e2e + resultType: e2e + spec: + containers: + - env: + - name: E2E_FOCUS + value: Conformance + image: gcr.io/heptio-images/kube-conformance:v1.8 + imagePullPolicy: Always + name: e2e + volumeMounts: + - mountPath: /tmp/results + name: results + - command: + - sh + - -c + - /sonobuoy worker global -v 5 --logtostderr + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: RESULTS_DIR + value: /tmp/results + image: gcr.io/heptio-images/sonobuoy:master + imagePullPolicy: Always + name: sonobuoy-worker + volumeMounts: + - mountPath: /etc/sonobuoy + name: config + - mountPath: /tmp/results + name: results + restartPolicy: Never + serviceAccountName: sonobuoy-serviceaccount + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + volumes: + - emptyDir: {} + name: results + - configMap: + name: __SONOBUOY_CONFIGMAP__ + name: config + systemdlogs.yaml: | + driver: DaemonSet + name: systemd_logs + resultType: systemd_logs + spec: + containers: + - command: + - sh + - -c + - /get_systemd_logs.sh && sleep 3600 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: RESULTS_DIR + value: /tmp/results + - name: CHROOT_DIR + value: /node + image: gcr.io/heptio-images/sonobuoy-plugin-systemd-logs:latest + imagePullPolicy: Always + name: systemd-logs + securityContext: + privileged: true + volumeMounts: + - mountPath: /node + name: root + - mountPath: /tmp/results + name: results + - mountPath: /etc/sonobuoy + name: config + - command: + - sh + - -c + - /sonobuoy worker single-node -v 5 --logtostderr && sleep 3600 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: RESULTS_DIR + value: /tmp/results + image: gcr.io/heptio-images/sonobuoy:master + imagePullPolicy: Always + name: sonobuoy-worker + securityContext: + privileged: true + volumeMounts: + - mountPath: /tmp/results + name: results + - mountPath: /etc/sonobuoy + name: config + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + hostPID: true + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + name: root + - emptyDir: {} + name: results + - configMap: + name: __SONOBUOY_CONFIGMAP__ + name: config +kind: ConfigMap +metadata: + labels: + component: sonobuoy + name: sonobuoy-plugins-cm + namespace: heptio-sonobuoy +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + component: sonobuoy + run: sonobuoy-master + tier: analysis + name: sonobuoy + namespace: heptio-sonobuoy +spec: + nodeSelector: + kubernetes.io/hostname: n0 + containers: + - command: + - /bin/bash + - -c + - /sonobuoy master -v 3 --logtostderr + env: + - name: SONOBUOY_ADVERTISE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: gcr.io/heptio-images/sonobuoy:master + imagePullPolicy: Always + name: kube-sonobuoy + volumeMounts: + - mountPath: /etc/sonobuoy + name: sonobuoy-config-volume + - mountPath: /plugins.d + name: sonobuoy-plugins-volume + - mountPath: /tmp/sonobuoy + name: output-volume + restartPolicy: Never + serviceAccountName: sonobuoy-serviceaccount + volumes: + - configMap: + name: sonobuoy-config-cm + name: sonobuoy-config-volume + - configMap: + name: sonobuoy-plugins-cm + name: sonobuoy-plugins-volume + - hostPath: + path: /mnt/sonobuoy + type: Directory + name: output-volume +--- +apiVersion: v1 +kind: Service +metadata: + labels: + component: sonobuoy + run: sonobuoy-master + name: sonobuoy-master + namespace: heptio-sonobuoy +spec: + ports: + - port: 8080 + protocol: TCP + targetPort: 8080 + selector: + run: sonobuoy-master + type: ClusterIP diff --git a/tools/g2/stages/conformance.sh b/tools/g2/stages/conformance.sh new file mode 100755 index 00000000..8c4629d6 --- /dev/null +++ b/tools/g2/stages/conformance.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +source ${GATE_UTILS} + +rm -rf ${WORKSPACE}/conformance +mkdir -p ${WORKSPACE}/conformance + +rsync_cmd ${WORKSPACE}/tools/g2/sonobuoy.yaml ${GENESIS_NAME}:/root/ +ssh_cmd ${GENESIS_NAME} mkdir -p /mnt/sonobuoy +kubectl_apply ${GENESIS_NAME} /root/sonobuoy.yaml + +kubectl_wait_for_pod ${GENESIS_NAME} heptio-sonobuoy sonobuoy 7200 + +FILENAME=$(ssh_cmd ${GENESIS_NAME} ls /mnt/sonobuoy) +rsync_cmd ${GENESIS_NAME}:/mnt/sonobuoy/${FILENAME} ${WORKSPACE}/conformance/sonobuoy.tgz +tar xf ${WORKSPACE}/conformance/sonobuoy.tgz -C ${WORKSPACE}/conformance + +tail -n 1 conformance/plugins/e2e/results/e2e.log | grep '^SUCCESS!' diff --git a/tools/gate/default-config-env b/tools/gate/default-config-env index b868f596..ee1fa2f9 100644 --- a/tools/gate/default-config-env +++ b/tools/gate/default-config-env @@ -3,7 +3,7 @@ IMAGE_CALICO_CNI=quay.io/calico/cni:v1.11.0 IMAGE_CALICO_CTL=quay.io/calico/ctl:v1.6.1 IMAGE_CALICO_KUBE_CONTROLLERS=quay.io/calico/kube-controllers:v1.0.0 IMAGE_CALICO_NODE=quay.io/calico/node:v2.6.1 -IMAGE_COREDNS=coredns/coredns:011 +IMAGE_COREDNS=coredns/coredns:0.9.9 IMAGE_ETCD=quay.io/coreos/etcd:v3.0.17 IMAGE_HELM=lachlanevenson/k8s-helm:v2.5.1 IMAGE_HYPERKUBE=gcr.io/google_containers/hyperkube-amd64:v1.8.0 diff --git a/tools/registry/IMAGES b/tools/registry/IMAGES index 65efd537..260f5ded 100644 --- a/tools/registry/IMAGES +++ b/tools/registry/IMAGES @@ -1,5 +1,5 @@ # source_name, tag, cache_name -coredns/coredns,011,coredns +coredns/coredns,0.9.9,coredns gcr.io/google_containers/hyperkube-amd64,v1.8.0,hyperkube gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64,1.14.4,k8s-dns-dnsmasq-nanny-amd64 gcr.io/google_containers/k8s-dns-kube-dns-amd64,1.14.4,k8s-dns-kube-dns-amd64