From 886007b36e1b42847f2908e660d31334ddabdf1f Mon Sep 17 00:00:00 2001 From: "Egorov, Stanislav (se6518)" Date: Thu, 11 Jul 2019 11:37:53 -0700 Subject: [PATCH] New CLI option to extract hyperkube New option --extract-hyperkube to declare the way how hyperkube will be delivered. By default this option is disabled which means hyperkube should be extracted before running promenade container for the first time. When it's enabled the appropriate env vars should be set for promenade container to be able to extract hyperkube binary from image. Change-Id: I2c45100e1e953d859d768ec80f268bd490ce3a81 --- doc/source/distribution.rst | 32 ++++++++++++++++++++++ doc/source/index.rst | 1 + promenade/builder.py | 29 ++++++++++++-------- promenade/cli.py | 9 +++++- promenade/config.py | 11 ++++++-- promenade/control/join_scripts.py | 3 ++ tools/g2/lib/config.sh | 3 +- tools/g2/stages/build-scripts.sh | 15 ++++------ tools/simple-deployment.sh | 18 ++++++------ tools/zuul/playbooks/deploy-promenade.yaml | 2 +- 10 files changed, 86 insertions(+), 37 deletions(-) create mode 100644 doc/source/distribution.rst diff --git a/doc/source/distribution.rst b/doc/source/distribution.rst new file mode 100644 index 00000000..e33b11d4 --- /dev/null +++ b/doc/source/distribution.rst @@ -0,0 +1,32 @@ +Distribution +============ + +Promenade is using Hyperkube for all Kubernetes components: kubelet, kubectl, etc. +By default Hyperkube binary should be extracted from the image before running Promenade. +This is done by external scripts and is not integrated into Promenade source code. +The other way is to let Promenade do the job and extract binary. This one is more complicated, +needs to share Docker socket inside Promenade container and is optional. + +Default behavior +---------------- + +IMAGE_HYPERKUBE should be exported and set to appropriate value. +Before running build-all CLI for Promenade need to run utility container which will copy binary from image to a shared location. +See tools/g2/stages/build-scripts.sh for reference. + + +Integrated solution +------------------- + +To let Promenade extract binary need to provide more env vars and shared locations for Promenade container. +Also need to enable option --extract-hyperkube in Promenade CLI. + +Define var for Docker socket(it should be available for user to read/write): +DOCKER_SOCK="/var/run/docker.sock" + +Provide it for container: +-v "${DOCKER_SOCK}:${DOCKER_SOCK}" +-e "DOCKER_HOST=unix:/${DOCKER_SOCK}" + +Provide additional var(it's for internal operations): +-e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}" diff --git a/doc/source/index.rst b/doc/source/index.rst index 8483e160..4eca5ccd 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,6 +32,7 @@ Promenade Configuration Guide developer-onboarding design + distribution getting-started configuration/index troubleshooting/index diff --git a/promenade/builder.py b/promenade/builder.py index 4922cfc9..ec4b082b 100644 --- a/promenade/builder.py +++ b/promenade/builder.py @@ -48,7 +48,7 @@ class Builder: if 'content' in file_spec: data = file_spec['content'] elif 'docker_image' in file_spec: - data = _fetch_image_content(self.config.container_info, + data = _fetch_image_content(self.config, file_spec['docker_image'], file_spec['file_path']) elif 'symlink' in file_spec: @@ -71,7 +71,6 @@ class Builder: self.config.get_path('Genesis:files', [])) def build_all(self, *, output_dir): - self.config.get_container_info() self.build_genesis(output_dir=output_dir) for node_document in self.config.iterate( schema='promenade/KubernetesNode/v1'): @@ -179,6 +178,7 @@ def _encrypt(cfg_dict, data): # The following environment variables should be used +# to extract hyperkube from image: # export DOCKER_HOST="unix://var/run/docker.sock" # export PROMENADE_TMP="tmp_dir_on_host" # export PROMENADE_TMP_LOCAL="tmp_dir_inside_container" @@ -187,20 +187,25 @@ def _encrypt(cfg_dict, data): @CACHE.cache('fetch_image', expire=72 * 3600) def _fetch_image_content(config, image_url, file_path): file_name = os.path.basename(file_path) - if config is None: - result_path = os.path.join(TMP_CACHE, file_name) - if not os.path.isfile(result_path): - raise Exception( - 'ERROR: there is no container info and no file in cache') - else: - result_path = os.path.join(config['dir_local'], file_name) - client = config['client'] - vol = {config['dir']: {'bind': config['dir_local'], 'mode': 'rw'}} - cmd = 'cp -v {} {}'.format(file_path, config['dir_local']) + if config.extract_hyperkube: + container_info = config.get_container_info() + result_path = os.path.join(container_info['dir_local'], file_name) + client = container_info['client'] + vol = { + container_info['dir']: { + 'bind': container_info['dir_local'], + 'mode': 'rw' + } + } + cmd = 'cp -v {} {}'.format(file_path, container_info['dir_local']) image = client.images.pull(image_url) output = client.containers.run( image, command=cmd, auto_remove=True, volumes=vol) LOG.debug(output) + else: + result_path = os.path.join(TMP_CACHE, file_name) + if not os.path.isfile(result_path): + raise Exception('ERROR: there is no hyperkube in cache') f = open(result_path, 'rb') return f.read() diff --git a/promenade/cli.py b/promenade/cli.py index d143da80..e7c42fdf 100644 --- a/promenade/cli.py +++ b/promenade/cli.py @@ -26,18 +26,25 @@ def promenade(*, verbose): required=True, help='Location to write complete cluster configuration.') @click.option('--validators', is_flag=True, help='Generate validation scripts') +@click.option( + '--extract-hyperkube', + is_flag=True, + default=False, + help='Extract hyperkube binary from image') @click.option( '--leave-kubectl', is_flag=True, help='Leave behind kubectl on joined nodes') @click.argument('config_files', nargs=-1, type=click.File('rb')) -def build_all(*, config_files, leave_kubectl, output_dir, validators): +def build_all(*, config_files, extract_hyperkube, leave_kubectl, output_dir, + validators): debug = _debug() try: c = config.Configuration.from_streams( debug=debug, substitute=True, allow_missing_substitutions=False, + extract_hyperkube=extract_hyperkube, leave_kubectl=leave_kubectl, streams=config_files) b = builder.Builder(c, validators=validators) diff --git a/promenade/config.py b/promenade/config.py index 7f2b1d6e..edf0bbec 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -21,6 +21,7 @@ class Configuration: debug=False, substitute=True, allow_missing_substitutions=True, + extract_hyperkube=True, leave_kubectl=False, validate=True): LOG.info("Parsing document schemas.") @@ -39,9 +40,9 @@ class Configuration: raise exceptions.DeckhandException(str(e)) LOG.info("Deckhand engine returned %d documents." % len(documents)) - self.container_info = None self.debug = debug self.documents = documents + self.extract_hyperkube = extract_hyperkube self.leave_kubectl = leave_kubectl if validate: @@ -116,6 +117,8 @@ class Configuration: for doc in self.iterate(*args, **kwargs): return doc + # try to use docker socket from ENV + # supported the same way like for docker client def get_container_info(self): LOG.debug( 'Getting access to Docker via socket and getting mount points') @@ -123,7 +126,7 @@ class Configuration: try: client.ping() except: - return + raise Exception('Docker is not responding, check ENV vars') tmp_dir = os.getenv('PROMENADE_TMP') if tmp_dir is None: raise Exception('ERROR: undefined PROMENADE_TMP') @@ -132,7 +135,7 @@ class Configuration: raise Exception('ERROR: undefined PROMENADE_TMP_LOCAL') if not os.path.exists(tmp_dir_local): raise Exception('ERROR: {} not found'.format(tmp_dir_local)) - self.container_info = { + return { 'client': client, 'dir': tmp_dir, 'dir_local': tmp_dir_local, @@ -150,6 +153,7 @@ class Configuration: return Configuration( debug=self.debug, documents=documents, + extract_hyperkube=self.extract_hyperkube, leave_kubectl=self.leave_kubectl, substitute=False, validate=False) @@ -173,6 +177,7 @@ class Configuration: return Configuration( debug=self.debug, documents=documents, + extract_hyperkube=self.extract_hyperkube, leave_kubectl=self.leave_kubectl, substitute=False, validate=False) diff --git a/promenade/control/join_scripts.py b/promenade/control/join_scripts.py index ca5948c3..0a56e34c 100644 --- a/promenade/control/join_scripts.py +++ b/promenade/control/join_scripts.py @@ -49,10 +49,13 @@ class JoinScriptsResource(BaseResource): join_ips = _get_join_ips() + # extract_hyperkube is False for join script because hyperkube should + # be extracted in the init container before running promenade try: config = Configuration.from_design_ref( design_ref, allow_missing_substitutions=False, + extract_hyperkube=False, leave_kubectl=leave_kubectl) except exceptions.DeckhandException: LOG.exception('Caught Deckhand render error for configuration') diff --git a/tools/g2/lib/config.sh b/tools/g2/lib/config.sh index b285dfd3..f1069533 100644 --- a/tools/g2/lib/config.sh +++ b/tools/g2/lib/config.sh @@ -2,11 +2,12 @@ export TEMP_DIR=${TEMP_DIR:-$(mktemp -d)} export BASE_IMAGE_SIZE=${BASE_IMAGE_SIZE:-68719476736} export BASE_IMAGE_URL=${BASE_IMAGE_URL:-https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img} export IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/airshipit/promenade:master} +export IMAGE_HYPERKUBE=${IMAGE_HYPERKUBE:-gcr.io/google_containers/hyperkube-amd64:v1.11.6} export NGINX_DIR="${TEMP_DIR}/nginx" export NGINX_URL="http://192.168.77.1:7777" export PROMENADE_BASE_URL="http://promenade-api.ucp.svc.cluster.local" export PROMENADE_DEBUG=${PROMENADE_DEBUG:-0} -export PROMENADE_TMP_LOCAL=${PROMENADE_TMP_LOCAL:-tmp_bin} +export PROMENADE_TMP_LOCAL=${PROMENADE_TMP_LOCAL:-cache} export PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY:-testkey} export REGISTRY_DATA_DIR=${REGISTRY_DATA_DIR:-/mnt/registry} export VIRSH_POOL=${VIRSH_POOL:-promenade} diff --git a/tools/g2/stages/build-scripts.sh b/tools/g2/stages/build-scripts.sh index c1756e51..c264cb49 100755 --- a/tools/g2/stages/build-scripts.sh +++ b/tools/g2/stages/build-scripts.sh @@ -13,18 +13,17 @@ PROMENADE_TMP="${TEMP_DIR}/${PROMENADE_TMP_LOCAL}" mkdir -p "$PROMENADE_TMP" chmod 777 "$PROMENADE_TMP" -DOCKER_SOCK="/var/run/docker.sock" -sudo chmod o+rw $DOCKER_SOCK +log Prepare hyperkube +docker run --rm -t \ + -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ + "${IMAGE_HYPERKUBE}" \ + cp /hyperkube "/tmp/${PROMENADE_TMP_LOCAL}" log Building scripts docker run --rm -t \ -w /target \ -v "${TEMP_DIR}:/target" \ - -v "${PROMENADE_TMP}:/${PROMENADE_TMP_LOCAL}" \ - -v "${DOCKER_SOCK}:${DOCKER_SOCK}" \ - -e "DOCKER_HOST=unix:/${DOCKER_SOCK}" \ - -e "PROMENADE_TMP=${PROMENADE_TMP}" \ - -e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}" \ + -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ -e "PROMENADE_DEBUG=${PROMENADE_DEBUG}" \ -e "PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY}" \ "${IMAGE_PROMENADE}" \ @@ -33,5 +32,3 @@ docker run --rm -t \ --validators \ -o scripts \ config/*.yaml - -sudo chmod o-rw $DOCKER_SOCK diff --git a/tools/simple-deployment.sh b/tools/simple-deployment.sh index 3689ac9c..2049b38e 100755 --- a/tools/simple-deployment.sh +++ b/tools/simple-deployment.sh @@ -3,6 +3,7 @@ set -eux IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/airshipit/promenade:master} +IMAGE_HYPERKUBE=${IMAGE_HYPERKUBE:-gcr.io/google_containers/hyperkube-amd64:v1.11.6} PROMENADE_DEBUG=${PROMENADE_DEBUG:-0} SCRIPT_DIR=$(realpath $(dirname $0)) @@ -29,9 +30,6 @@ PROMENADE_TMP="${SCRIPT_DIR}/${PROMENADE_TMP_LOCAL}" mkdir -p "$PROMENADE_TMP" chmod 777 "$PROMENADE_TMP" -DOCKER_SOCK="/var/run/docker.sock" -sudo chmod o+rw $DOCKER_SOCK - cp "${CONFIG_SOURCE}"/*.yaml ${BUILD_DIR} if [ ${REPLACE} == 'replace' ] @@ -59,6 +57,12 @@ docker run --rm -t \ fi if [[ -z $1 ]] || [[ $1 = build-all ]]; then +echo === Prepare hyperkube === +docker run --rm -t \ + -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ + "${IMAGE_HYPERKUBE}" \ + cp /hyperkube "/tmp/${PROMENADE_TMP_LOCAL}" + echo === Building bootstrap scripts === docker run --rm -t \ -w /target \ @@ -66,11 +70,7 @@ docker run --rm -t \ -e http_proxy=${HTTP_PROXY} \ -e https_proxy=${HTTPS_PROXY} \ -e no_proxy=${NO_PROXY} \ - -v "${PROMENADE_TMP}:/${PROMENADE_TMP_LOCAL}" \ - -v "${DOCKER_SOCK}:${DOCKER_SOCK}" \ - -e "DOCKER_HOST=unix:/${DOCKER_SOCK}" \ - -e "PROMENADE_TMP=${PROMENADE_TMP}" \ - -e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}" \ + -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ -v ${BUILD_DIR}:/target \ ${IMAGE_PROMENADE} \ promenade \ @@ -80,6 +80,4 @@ docker run --rm -t \ $(ls ${BUILD_DIR}) fi -sudo chmod o-rw $DOCKER_SOCK - echo === Done === diff --git a/tools/zuul/playbooks/deploy-promenade.yaml b/tools/zuul/playbooks/deploy-promenade.yaml index a5890c2d..1bfa21e4 100644 --- a/tools/zuul/playbooks/deploy-promenade.yaml +++ b/tools/zuul/playbooks/deploy-promenade.yaml @@ -16,7 +16,7 @@ HTTP_PROXY: "" HTTPS_PROXY: "" NO_PROXY: "" - PROMENADE_TMP_LOCAL: "tmp_bin" + PROMENADE_TMP_LOCAL: "cache" become: true tasks: - name: Install docker