From c7e72942a951ac8e81d29246327a7f7b1a71c346 Mon Sep 17 00:00:00 2001 From: Phil Sphicas Date: Thu, 14 Jan 2021 19:52:32 +0000 Subject: [PATCH] Remove hyperkube extraction functionality The extraction of the monolithic hyperkube binary from its container image to be used as kubelet was last relevant in Kubernetes 1.16. Since then, the hyperkube image has been deprecated, the structure of the image has been changed, and it has ultimately been eliminated in Kubernetes 1.19. This change cleans up promenade accordingly. Reverts the following commits: * 886007b New CLI option to extract hyperkube * 32a6c15 hyperkube image in promenade init * 955deed New source for hyperkube binary definition Change-Id: Ib62ecdf1af13abe8202a4ba4f86c39b9042ed13f --- .../promenade/templates/deployment-api.yaml | 21 +---------- charts/promenade/values.yaml | 1 - doc/source/distribution.rst | 32 ---------------- doc/source/index.rst | 1 - promenade/builder.py | 37 ------------------- promenade/cli.py | 9 +---- promenade/config.py | 30 --------------- promenade/control/join_scripts.py | 3 -- promenade/schemas/HostSystem.yaml | 10 ----- requirements-direct.txt | 1 - requirements-frozen.txt | 2 - tools/g2/lib/config.sh | 2 - tools/g2/stages/build-scripts.sh | 12 ------ tools/simple-deployment.sh | 14 ------- tools/zuul/playbooks/deploy-promenade.yaml | 1 - 15 files changed, 2 insertions(+), 174 deletions(-) delete mode 100644 doc/source/distribution.rst diff --git a/charts/promenade/templates/deployment-api.yaml b/charts/promenade/templates/deployment-api.yaml index c67bf3d9..c597051f 100644 --- a/charts/promenade/templates/deployment-api.yaml +++ b/charts/promenade/templates/deployment-api.yaml @@ -41,7 +41,7 @@ spec: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} -{{ dict "envAll" $envAll "podName" "promenade-api" "containerNames" (list "promenade-util" "promenade-api") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ dict "envAll" $envAll "podName" "promenade-api" "containerNames" (list "promenade-api") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} spec: {{ dict "envAll" $envAll "application" "promenade" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} affinity: @@ -50,25 +50,6 @@ spec: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} serviceAccountName: promenade terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.api.timeout | default "30" }} - initContainers: - - name: promenade-util -{{ dict "envAll" $envAll "application" "promenade" "container" "promenade_util" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 8 }} - command: -{{- if $mounts_promenade_api.volumeMounts }} - - "cp" - - "/hyperkube" - {{- range $mounts_promenade_api.volumeMounts }} - {{- if eq "cache" (index . "name") }} - - {{ index . "mountPath" | quote }} - {{- end }} - {{- end }} -{{- else }} - - "true" -{{- end }} - image: {{ .Values.images.tags.hyperkube }} - imagePullPolicy: IfNotPresent - volumeMounts: -{{ if $mounts_promenade_api.volumeMounts }}{{ toYaml $mounts_promenade_api.volumeMounts | indent 8 }}{{ end }} containers: - name: promenade-api image: {{ .Values.images.tags.promenade }} diff --git a/charts/promenade/values.yaml b/charts/promenade/values.yaml index f73c625b..95675ad4 100644 --- a/charts/promenade/values.yaml +++ b/charts/promenade/values.yaml @@ -49,7 +49,6 @@ conf: images: tags: monitoring_image: busybox:1.28.3 - hyperkube: k8s.gcr.io/hyperkube-amd64:v1.18.6 promenade: quay.io/airshipit/promenade:latest ks_user: docker.io/openstackhelm/heat:newton ks_service: docker.io/openstackhelm/heat:newton diff --git a/doc/source/distribution.rst b/doc/source/distribution.rst deleted file mode 100644 index e33b11d4..00000000 --- a/doc/source/distribution.rst +++ /dev/null @@ -1,32 +0,0 @@ -Distribution -============ - -Promenade is using Hyperkube for all Kubernetes components: kubelet, kubectl, etc. -By default Hyperkube binary should be extracted from the image before running Promenade. -This is done by external scripts and is not integrated into Promenade source code. -The other way is to let Promenade do the job and extract binary. This one is more complicated, -needs to share Docker socket inside Promenade container and is optional. - -Default behavior ----------------- - -IMAGE_HYPERKUBE should be exported and set to appropriate value. -Before running build-all CLI for Promenade need to run utility container which will copy binary from image to a shared location. -See tools/g2/stages/build-scripts.sh for reference. - - -Integrated solution -------------------- - -To let Promenade extract binary need to provide more env vars and shared locations for Promenade container. -Also need to enable option --extract-hyperkube in Promenade CLI. - -Define var for Docker socket(it should be available for user to read/write): -DOCKER_SOCK="/var/run/docker.sock" - -Provide it for container: --v "${DOCKER_SOCK}:${DOCKER_SOCK}" --e "DOCKER_HOST=unix:/${DOCKER_SOCK}" - -Provide additional var(it's for internal operations): --e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}" diff --git a/doc/source/index.rst b/doc/source/index.rst index 4eca5ccd..8483e160 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,7 +32,6 @@ Promenade Configuration Guide developer-onboarding design - distribution getting-started configuration/index troubleshooting/index diff --git a/promenade/builder.py b/promenade/builder.py index 5ca7eeea..bd37bb87 100644 --- a/promenade/builder.py +++ b/promenade/builder.py @@ -47,10 +47,6 @@ class Builder: islink = False if 'content' in file_spec: data = file_spec['content'] - elif 'docker_image' in file_spec: - data = _fetch_image_content(self.config, - file_spec['docker_image'], - file_spec['file_path']) elif 'symlink' in file_spec: data = file_spec['symlink'] islink = True @@ -179,39 +175,6 @@ def _encrypt(cfg_dict, data): decrypt_teardown_command) -# The following environment variables should be used -# to extract hyperkube from image: -# export DOCKER_HOST="unix://var/run/docker.sock" -# export PROMENADE_TMP="tmp_dir_on_host" -# export PROMENADE_TMP_LOCAL="tmp_dir_inside_container" -# PROMENADE_TMP is the full path to temp dir from host -# inside promenade container it should be bind to PROMENADE_TMP_LOCAL -@CACHE.cache('fetch_image', expire=72 * 3600) -def _fetch_image_content(config, image_url, file_path): - file_name = os.path.basename(file_path) - if config.extract_hyperkube: - container_info = config.get_container_info() - result_path = os.path.join(container_info['dir_local'], file_name) - client = container_info['client'] - vol = { - container_info['dir']: { - 'bind': container_info['dir_local'], - 'mode': 'rw' - } - } - cmd = 'cp -v {} {}'.format(file_path, container_info['dir_local']) - image = client.images.pull(image_url) - output = client.containers.run( - image, command=cmd, auto_remove=True, volumes=vol) - LOG.debug(output) - else: - result_path = os.path.join(TMP_CACHE, file_name) - if not os.path.isfile(result_path): - raise Exception('ERROR: there is no hyperkube in cache') - f = open(result_path, 'rb') - return f.read() - - @CACHE.cache('fetch_tarball_content', expire=72 * 3600) def _fetch_tar_content(url, path): content = _fetch_tar_url(url) diff --git a/promenade/cli.py b/promenade/cli.py index 6d06a860..30d9ffb7 100644 --- a/promenade/cli.py +++ b/promenade/cli.py @@ -27,25 +27,18 @@ def promenade(*, verbose): required=True, help='Location to write complete cluster configuration.') @click.option('--validators', is_flag=True, help='Generate validation scripts') -@click.option( - '--extract-hyperkube', - is_flag=True, - default=False, - help='Extract hyperkube binary from image') @click.option( '--leave-kubectl', is_flag=True, help='Leave behind kubectl on joined nodes') @click.argument('config_files', nargs=-1, type=click.File('rb')) -def build_all(*, config_files, extract_hyperkube, leave_kubectl, output_dir, - validators): +def build_all(*, config_files, leave_kubectl, output_dir, validators): debug = _debug() try: c = config.Configuration.from_streams( debug=debug, substitute=True, allow_missing_substitutions=False, - extract_hyperkube=extract_hyperkube, leave_kubectl=leave_kubectl, streams=config_files) b = builder.Builder(c, validators=validators) diff --git a/promenade/config.py b/promenade/config.py index c9e24a39..791cd357 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -1,9 +1,7 @@ from . import exceptions, logging, validation from . import design_ref as dr -import docker import jinja2 import jsonpath_ng -import os import yaml from deckhand.engine import layering @@ -21,7 +19,6 @@ class Configuration: debug=False, substitute=True, allow_missing_substitutions=True, - extract_hyperkube=True, leave_kubectl=False, validate=True): LOG.info("Parsing document schemas.") @@ -42,7 +39,6 @@ class Configuration: LOG.info("Deckhand engine returned %d documents." % len(documents)) self.debug = debug self.documents = documents - self.extract_hyperkube = extract_hyperkube self.leave_kubectl = leave_kubectl if validate: @@ -117,30 +113,6 @@ class Configuration: for doc in self.iterate(*args, **kwargs): return doc - # try to use docker socket from ENV - # supported the same way like for docker client - def get_container_info(self): - LOG.debug( - 'Getting access to Docker via socket and getting mount points') - client = docker.from_env() - try: - client.ping() - except Exception: - raise Exception('Docker is not responding, check ENV vars') - tmp_dir = os.getenv('PROMENADE_TMP') - if tmp_dir is None: - raise Exception('ERROR: undefined PROMENADE_TMP') - tmp_dir_local = os.getenv('PROMENADE_TMP_LOCAL') - if tmp_dir_local is None: - raise Exception('ERROR: undefined PROMENADE_TMP_LOCAL') - if not os.path.exists(tmp_dir_local): - raise Exception('ERROR: {} not found'.format(tmp_dir_local)) - return { - 'client': client, - 'dir': tmp_dir, - 'dir_local': tmp_dir_local, - } - def extract_genesis_config(self): LOG.debug('Extracting genesis config.') documents = [] @@ -153,7 +125,6 @@ class Configuration: return Configuration( debug=self.debug, documents=documents, - extract_hyperkube=self.extract_hyperkube, leave_kubectl=self.leave_kubectl, substitute=False, validate=False) @@ -177,7 +148,6 @@ class Configuration: return Configuration( debug=self.debug, documents=documents, - extract_hyperkube=self.extract_hyperkube, leave_kubectl=self.leave_kubectl, substitute=False, validate=False) diff --git a/promenade/control/join_scripts.py b/promenade/control/join_scripts.py index 0a56e34c..ca5948c3 100644 --- a/promenade/control/join_scripts.py +++ b/promenade/control/join_scripts.py @@ -49,13 +49,10 @@ class JoinScriptsResource(BaseResource): join_ips = _get_join_ips() - # extract_hyperkube is False for join script because hyperkube should - # be extracted in the init container before running promenade try: config = Configuration.from_design_ref( design_ref, allow_missing_substitutions=False, - extract_hyperkube=False, leave_kubectl=leave_kubectl) except exceptions.DeckhandException: LOG.exception('Caught Deckhand render error for configuration') diff --git a/promenade/schemas/HostSystem.yaml b/promenade/schemas/HostSystem.yaml index 4fb15cd6..1749d792 100644 --- a/promenade/schemas/HostSystem.yaml +++ b/promenade/schemas/HostSystem.yaml @@ -43,10 +43,6 @@ data: $ref: '#/definitions/url' tar_path: $ref: '#/definitions/rel_path' - docker_image: - $ref: '#/definitions/url' - file_path: - $ref: '#/definitions/abs_path' symlink: $ref: '#/definitions/abs_path' required: @@ -68,12 +64,6 @@ data: required: - tar_url - tar_path - - type: object - allOf: - - type: object - required: - - docker_image - - file_path additionalProperties: false image: type: string diff --git a/requirements-direct.txt b/requirements-direct.txt index 6fb6ccb2..23ba5db8 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -1,6 +1,5 @@ beaker==1.10.0 click==6.7 -docker==3.7.2 falcon==1.4.1 jinja2==2.10 jsonpath-ng==1.4.3 diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 204050bf..812bc8b9 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -16,8 +16,6 @@ Deckhand @ git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65e decorator==4.4.2 deepdiff==3.3.0 dnspython==1.16.0 -docker==3.7.2 -docker-pycreds==0.4.0 dogpile.cache==0.9.2 eventlet==0.25.2 extras==1.0.0 diff --git a/tools/g2/lib/config.sh b/tools/g2/lib/config.sh index dc414fd3..e17aef64 100644 --- a/tools/g2/lib/config.sh +++ b/tools/g2/lib/config.sh @@ -3,12 +3,10 @@ export BASE_IMAGE_SIZE=${BASE_IMAGE_SIZE:-344784896} export BASE_IMAGE_URL=${BASE_IMAGE_URL:-https://cloud-images.ubuntu.com/releases/bionic/release/ubuntu-18.04-server-cloudimg-amd64.img} export IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/airshipit/promenade:master} export IMAGE_PROMENADE_DISTRO=${IMAGE_PROMENADE_DISTRO:-ubuntu_bionic} -export IMAGE_HYPERKUBE=${IMAGE_HYPERKUBE:-k8s.gcr.io/hyperkube-amd64:v1.18.6} export NGINX_DIR="${TEMP_DIR}/nginx" export NGINX_URL="http://192.168.77.1:7777" export PROMENADE_BASE_URL="http://promenade-api.ucp.svc.cluster.local" export PROMENADE_DEBUG=${PROMENADE_DEBUG:-0} -export PROMENADE_TMP_LOCAL=${PROMENADE_TMP_LOCAL:-cache} export PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY:-testkey} export REGISTRY_DATA_DIR=${REGISTRY_DATA_DIR:-/mnt/registry} export VIRSH_POOL=${VIRSH_POOL:-promenade} diff --git a/tools/g2/stages/build-scripts.sh b/tools/g2/stages/build-scripts.sh index c264cb49..22fa3920 100755 --- a/tools/g2/stages/build-scripts.sh +++ b/tools/g2/stages/build-scripts.sh @@ -8,22 +8,10 @@ cd "${TEMP_DIR}" mkdir scripts chmod 777 scripts -PROMENADE_TMP_LOCAL="$(basename "$PROMENADE_TMP_LOCAL")" -PROMENADE_TMP="${TEMP_DIR}/${PROMENADE_TMP_LOCAL}" -mkdir -p "$PROMENADE_TMP" -chmod 777 "$PROMENADE_TMP" - -log Prepare hyperkube -docker run --rm -t \ - -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ - "${IMAGE_HYPERKUBE}" \ - cp /hyperkube "/tmp/${PROMENADE_TMP_LOCAL}" - log Building scripts docker run --rm -t \ -w /target \ -v "${TEMP_DIR}:/target" \ - -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ -e "PROMENADE_DEBUG=${PROMENADE_DEBUG}" \ -e "PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY}" \ "${IMAGE_PROMENADE}" \ diff --git a/tools/simple-deployment.sh b/tools/simple-deployment.sh index 63f57b6d..574b3f44 100755 --- a/tools/simple-deployment.sh +++ b/tools/simple-deployment.sh @@ -3,7 +3,6 @@ set -eux IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/airshipit/promenade:master} -IMAGE_HYPERKUBE=${IMAGE_HYPERKUBE:-k8s.gcr.io/hyperkube-amd64:v1.18.6} PROMENADE_DEBUG=${PROMENADE_DEBUG:-0} SCRIPT_DIR=$(realpath $(dirname $0)) @@ -25,11 +24,6 @@ rm -rf ${BUILD_DIR} mkdir -p ${BUILD_DIR} chmod 777 ${BUILD_DIR} -PROMENADE_TMP_LOCAL="$(basename "$PROMENADE_TMP_LOCAL")" -PROMENADE_TMP="${SCRIPT_DIR}/${PROMENADE_TMP_LOCAL}" -mkdir -p "$PROMENADE_TMP" -chmod 777 "$PROMENADE_TMP" - cp "${CONFIG_SOURCE}"/*.yaml ${BUILD_DIR} if [ ${REPLACE} == 'replace' ] @@ -57,13 +51,6 @@ docker run --rm -t \ fi if [[ -z $1 ]] || [[ $1 = build-all ]]; then -echo === Prepare hyperkube === -docker run --rm -t \ - -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ - --entrypoint 'cp' \ - "${IMAGE_HYPERKUBE}" \ - /hyperkube "/tmp/${PROMENADE_TMP_LOCAL}" - echo === Building bootstrap scripts === docker run --rm -t \ -w /target \ @@ -71,7 +58,6 @@ docker run --rm -t \ -e http_proxy=${HTTP_PROXY} \ -e https_proxy=${HTTPS_PROXY} \ -e no_proxy=${NO_PROXY} \ - -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ -v ${BUILD_DIR}:/target \ ${IMAGE_PROMENADE} \ promenade \ diff --git a/tools/zuul/playbooks/deploy-promenade.yaml b/tools/zuul/playbooks/deploy-promenade.yaml index 56ad1fa7..2d1be239 100644 --- a/tools/zuul/playbooks/deploy-promenade.yaml +++ b/tools/zuul/playbooks/deploy-promenade.yaml @@ -16,7 +16,6 @@ HTTP_PROXY: "" HTTPS_PROXY: "" NO_PROXY: "" - PROMENADE_TMP_LOCAL: "cache" DISTRO: "{{ distro }}" become: true tasks: