diff --git a/doc/source/distribution.rst b/doc/source/distribution.rst new file mode 100644 index 00000000..e33b11d4 --- /dev/null +++ b/doc/source/distribution.rst @@ -0,0 +1,32 @@ +Distribution +============ + +Promenade is using Hyperkube for all Kubernetes components: kubelet, kubectl, etc. +By default Hyperkube binary should be extracted from the image before running Promenade. +This is done by external scripts and is not integrated into Promenade source code. +The other way is to let Promenade do the job and extract binary. This one is more complicated, +needs to share Docker socket inside Promenade container and is optional. + +Default behavior +---------------- + +IMAGE_HYPERKUBE should be exported and set to appropriate value. +Before running build-all CLI for Promenade need to run utility container which will copy binary from image to a shared location. +See tools/g2/stages/build-scripts.sh for reference. + + +Integrated solution +------------------- + +To let Promenade extract binary need to provide more env vars and shared locations for Promenade container. +Also need to enable option --extract-hyperkube in Promenade CLI. + +Define var for Docker socket(it should be available for user to read/write): +DOCKER_SOCK="/var/run/docker.sock" + +Provide it for container: +-v "${DOCKER_SOCK}:${DOCKER_SOCK}" +-e "DOCKER_HOST=unix:/${DOCKER_SOCK}" + +Provide additional var(it's for internal operations): +-e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}" diff --git a/doc/source/index.rst b/doc/source/index.rst index 8483e160..4eca5ccd 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,6 +32,7 @@ Promenade Configuration Guide developer-onboarding design + distribution getting-started configuration/index troubleshooting/index diff --git a/promenade/builder.py b/promenade/builder.py index 4922cfc9..ec4b082b 100644 --- a/promenade/builder.py +++ b/promenade/builder.py @@ -48,7 +48,7 @@ class Builder: if 'content' in file_spec: data = file_spec['content'] elif 'docker_image' in file_spec: - data = _fetch_image_content(self.config.container_info, + data = _fetch_image_content(self.config, file_spec['docker_image'], file_spec['file_path']) elif 'symlink' in file_spec: @@ -71,7 +71,6 @@ class Builder: self.config.get_path('Genesis:files', [])) def build_all(self, *, output_dir): - self.config.get_container_info() self.build_genesis(output_dir=output_dir) for node_document in self.config.iterate( schema='promenade/KubernetesNode/v1'): @@ -179,6 +178,7 @@ def _encrypt(cfg_dict, data): # The following environment variables should be used +# to extract hyperkube from image: # export DOCKER_HOST="unix://var/run/docker.sock" # export PROMENADE_TMP="tmp_dir_on_host" # export PROMENADE_TMP_LOCAL="tmp_dir_inside_container" @@ -187,20 +187,25 @@ def _encrypt(cfg_dict, data): @CACHE.cache('fetch_image', expire=72 * 3600) def _fetch_image_content(config, image_url, file_path): file_name = os.path.basename(file_path) - if config is None: - result_path = os.path.join(TMP_CACHE, file_name) - if not os.path.isfile(result_path): - raise Exception( - 'ERROR: there is no container info and no file in cache') - else: - result_path = os.path.join(config['dir_local'], file_name) - client = config['client'] - vol = {config['dir']: {'bind': config['dir_local'], 'mode': 'rw'}} - cmd = 'cp -v {} {}'.format(file_path, config['dir_local']) + if config.extract_hyperkube: + container_info = config.get_container_info() + result_path = os.path.join(container_info['dir_local'], file_name) + client = container_info['client'] + vol = { + container_info['dir']: { + 'bind': container_info['dir_local'], + 'mode': 'rw' + } + } + cmd = 'cp -v {} {}'.format(file_path, container_info['dir_local']) image = client.images.pull(image_url) output = client.containers.run( image, command=cmd, auto_remove=True, volumes=vol) LOG.debug(output) + else: + result_path = os.path.join(TMP_CACHE, file_name) + if not os.path.isfile(result_path): + raise Exception('ERROR: there is no hyperkube in cache') f = open(result_path, 'rb') return f.read() diff --git a/promenade/cli.py b/promenade/cli.py index d143da80..e7c42fdf 100644 --- a/promenade/cli.py +++ b/promenade/cli.py @@ -26,18 +26,25 @@ def promenade(*, verbose): required=True, help='Location to write complete cluster configuration.') @click.option('--validators', is_flag=True, help='Generate validation scripts') +@click.option( + '--extract-hyperkube', + is_flag=True, + default=False, + help='Extract hyperkube binary from image') @click.option( '--leave-kubectl', is_flag=True, help='Leave behind kubectl on joined nodes') @click.argument('config_files', nargs=-1, type=click.File('rb')) -def build_all(*, config_files, leave_kubectl, output_dir, validators): +def build_all(*, config_files, extract_hyperkube, leave_kubectl, output_dir, + validators): debug = _debug() try: c = config.Configuration.from_streams( debug=debug, substitute=True, allow_missing_substitutions=False, + extract_hyperkube=extract_hyperkube, leave_kubectl=leave_kubectl, streams=config_files) b = builder.Builder(c, validators=validators) diff --git a/promenade/config.py b/promenade/config.py index 7f2b1d6e..edf0bbec 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -21,6 +21,7 @@ class Configuration: debug=False, substitute=True, allow_missing_substitutions=True, + extract_hyperkube=True, leave_kubectl=False, validate=True): LOG.info("Parsing document schemas.") @@ -39,9 +40,9 @@ class Configuration: raise exceptions.DeckhandException(str(e)) LOG.info("Deckhand engine returned %d documents." % len(documents)) - self.container_info = None self.debug = debug self.documents = documents + self.extract_hyperkube = extract_hyperkube self.leave_kubectl = leave_kubectl if validate: @@ -116,6 +117,8 @@ class Configuration: for doc in self.iterate(*args, **kwargs): return doc + # try to use docker socket from ENV + # supported the same way like for docker client def get_container_info(self): LOG.debug( 'Getting access to Docker via socket and getting mount points') @@ -123,7 +126,7 @@ class Configuration: try: client.ping() except: - return + raise Exception('Docker is not responding, check ENV vars') tmp_dir = os.getenv('PROMENADE_TMP') if tmp_dir is None: raise Exception('ERROR: undefined PROMENADE_TMP') @@ -132,7 +135,7 @@ class Configuration: raise Exception('ERROR: undefined PROMENADE_TMP_LOCAL') if not os.path.exists(tmp_dir_local): raise Exception('ERROR: {} not found'.format(tmp_dir_local)) - self.container_info = { + return { 'client': client, 'dir': tmp_dir, 'dir_local': tmp_dir_local, @@ -150,6 +153,7 @@ class Configuration: return Configuration( debug=self.debug, documents=documents, + extract_hyperkube=self.extract_hyperkube, leave_kubectl=self.leave_kubectl, substitute=False, validate=False) @@ -173,6 +177,7 @@ class Configuration: return Configuration( debug=self.debug, documents=documents, + extract_hyperkube=self.extract_hyperkube, leave_kubectl=self.leave_kubectl, substitute=False, validate=False) diff --git a/promenade/control/join_scripts.py b/promenade/control/join_scripts.py index ca5948c3..0a56e34c 100644 --- a/promenade/control/join_scripts.py +++ b/promenade/control/join_scripts.py @@ -49,10 +49,13 @@ class JoinScriptsResource(BaseResource): join_ips = _get_join_ips() + # extract_hyperkube is False for join script because hyperkube should + # be extracted in the init container before running promenade try: config = Configuration.from_design_ref( design_ref, allow_missing_substitutions=False, + extract_hyperkube=False, leave_kubectl=leave_kubectl) except exceptions.DeckhandException: LOG.exception('Caught Deckhand render error for configuration') diff --git a/tools/g2/lib/config.sh b/tools/g2/lib/config.sh index b285dfd3..f1069533 100644 --- a/tools/g2/lib/config.sh +++ b/tools/g2/lib/config.sh @@ -2,11 +2,12 @@ export TEMP_DIR=${TEMP_DIR:-$(mktemp -d)} export BASE_IMAGE_SIZE=${BASE_IMAGE_SIZE:-68719476736} export BASE_IMAGE_URL=${BASE_IMAGE_URL:-https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img} export IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/airshipit/promenade:master} +export IMAGE_HYPERKUBE=${IMAGE_HYPERKUBE:-gcr.io/google_containers/hyperkube-amd64:v1.11.6} export NGINX_DIR="${TEMP_DIR}/nginx" export NGINX_URL="http://192.168.77.1:7777" export PROMENADE_BASE_URL="http://promenade-api.ucp.svc.cluster.local" export PROMENADE_DEBUG=${PROMENADE_DEBUG:-0} -export PROMENADE_TMP_LOCAL=${PROMENADE_TMP_LOCAL:-tmp_bin} +export PROMENADE_TMP_LOCAL=${PROMENADE_TMP_LOCAL:-cache} export PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY:-testkey} export REGISTRY_DATA_DIR=${REGISTRY_DATA_DIR:-/mnt/registry} export VIRSH_POOL=${VIRSH_POOL:-promenade} diff --git a/tools/g2/stages/build-scripts.sh b/tools/g2/stages/build-scripts.sh index c1756e51..c264cb49 100755 --- a/tools/g2/stages/build-scripts.sh +++ b/tools/g2/stages/build-scripts.sh @@ -13,18 +13,17 @@ PROMENADE_TMP="${TEMP_DIR}/${PROMENADE_TMP_LOCAL}" mkdir -p "$PROMENADE_TMP" chmod 777 "$PROMENADE_TMP" -DOCKER_SOCK="/var/run/docker.sock" -sudo chmod o+rw $DOCKER_SOCK +log Prepare hyperkube +docker run --rm -t \ + -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ + "${IMAGE_HYPERKUBE}" \ + cp /hyperkube "/tmp/${PROMENADE_TMP_LOCAL}" log Building scripts docker run --rm -t \ -w /target \ -v "${TEMP_DIR}:/target" \ - -v "${PROMENADE_TMP}:/${PROMENADE_TMP_LOCAL}" \ - -v "${DOCKER_SOCK}:${DOCKER_SOCK}" \ - -e "DOCKER_HOST=unix:/${DOCKER_SOCK}" \ - -e "PROMENADE_TMP=${PROMENADE_TMP}" \ - -e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}" \ + -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ -e "PROMENADE_DEBUG=${PROMENADE_DEBUG}" \ -e "PROMENADE_ENCRYPTION_KEY=${PROMENADE_ENCRYPTION_KEY}" \ "${IMAGE_PROMENADE}" \ @@ -33,5 +32,3 @@ docker run --rm -t \ --validators \ -o scripts \ config/*.yaml - -sudo chmod o-rw $DOCKER_SOCK diff --git a/tools/simple-deployment.sh b/tools/simple-deployment.sh index 3689ac9c..2049b38e 100755 --- a/tools/simple-deployment.sh +++ b/tools/simple-deployment.sh @@ -3,6 +3,7 @@ set -eux IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/airshipit/promenade:master} +IMAGE_HYPERKUBE=${IMAGE_HYPERKUBE:-gcr.io/google_containers/hyperkube-amd64:v1.11.6} PROMENADE_DEBUG=${PROMENADE_DEBUG:-0} SCRIPT_DIR=$(realpath $(dirname $0)) @@ -29,9 +30,6 @@ PROMENADE_TMP="${SCRIPT_DIR}/${PROMENADE_TMP_LOCAL}" mkdir -p "$PROMENADE_TMP" chmod 777 "$PROMENADE_TMP" -DOCKER_SOCK="/var/run/docker.sock" -sudo chmod o+rw $DOCKER_SOCK - cp "${CONFIG_SOURCE}"/*.yaml ${BUILD_DIR} if [ ${REPLACE} == 'replace' ] @@ -59,6 +57,12 @@ docker run --rm -t \ fi if [[ -z $1 ]] || [[ $1 = build-all ]]; then +echo === Prepare hyperkube === +docker run --rm -t \ + -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ + "${IMAGE_HYPERKUBE}" \ + cp /hyperkube "/tmp/${PROMENADE_TMP_LOCAL}" + echo === Building bootstrap scripts === docker run --rm -t \ -w /target \ @@ -66,11 +70,7 @@ docker run --rm -t \ -e http_proxy=${HTTP_PROXY} \ -e https_proxy=${HTTPS_PROXY} \ -e no_proxy=${NO_PROXY} \ - -v "${PROMENADE_TMP}:/${PROMENADE_TMP_LOCAL}" \ - -v "${DOCKER_SOCK}:${DOCKER_SOCK}" \ - -e "DOCKER_HOST=unix:/${DOCKER_SOCK}" \ - -e "PROMENADE_TMP=${PROMENADE_TMP}" \ - -e "PROMENADE_TMP_LOCAL=/${PROMENADE_TMP_LOCAL}" \ + -v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \ -v ${BUILD_DIR}:/target \ ${IMAGE_PROMENADE} \ promenade \ @@ -80,6 +80,4 @@ docker run --rm -t \ $(ls ${BUILD_DIR}) fi -sudo chmod o-rw $DOCKER_SOCK - echo === Done === diff --git a/tools/zuul/playbooks/deploy-promenade.yaml b/tools/zuul/playbooks/deploy-promenade.yaml index a5890c2d..1bfa21e4 100644 --- a/tools/zuul/playbooks/deploy-promenade.yaml +++ b/tools/zuul/playbooks/deploy-promenade.yaml @@ -16,7 +16,7 @@ HTTP_PROXY: "" HTTPS_PROXY: "" NO_PROXY: "" - PROMENADE_TMP_LOCAL: "tmp_bin" + PROMENADE_TMP_LOCAL: "cache" become: true tasks: - name: Install docker