From 8a393dfc808692c8b1d9fc43191d0e65ebfdbb36 Mon Sep 17 00:00:00 2001 From: "anthony.lin" Date: Mon, 28 May 2018 14:08:52 +0800 Subject: [PATCH] Add docker image publishing 1) Refactor shipyard image build 2) Add shipyard docker image publishing Change-Id: I1a007ae21a88d6468bf42724e5def5cd04ebe48e --- .zuul.yaml | 50 ++++++- Makefile | 25 +++- tools/gate/playbooks/osh-infra-build.yaml | 12 -- .../build-images/tasks/airship-shipyard.yaml | 104 +++++++++------ tools/image_tags.py | 126 ++++++++++++++++++ 5 files changed, 260 insertions(+), 57 deletions(-) create mode 100644 tools/image_tags.py diff --git a/.zuul.yaml b/.zuul.yaml index af74b34b..b3d493c2 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -33,6 +33,9 @@ - airship-shipyard-pep8 - airship-shipyard-security-bandit - airship-shipyard-image + post: + jobs: + - airship-shipyard-docker-build-post - nodeset: name: airship-shipyard-single-node @@ -130,6 +133,7 @@ - tools/gate/playbooks/osh-infra-upgrade-host.yaml - tools/gate/playbooks/osh-infra-deploy-docker.yaml run: tools/gate/playbooks/run-image.yaml + nodeset: airship-shipyard-single-node required-projects: - openstack/openstack-helm-infra irrelevant-files: @@ -142,4 +146,48 @@ description: | Run shipyard-image build. parent: airship-shipyard-image-base - nodeset: airship-shipyard-single-node + vars: + publish: false + tags: + dynamic: + patch_set: true + +- job: + name: airship-shipyard-docker-build-post + parent: airship-shipyard-image-base + secrets: + - airship_shipyard_quay_creds + vars: + publish: true + tags: + dynamic: + branch: true + commit: true + static: + - latest + +- secret: + name: airship_shipyard_quay_creds + data: + username: !encrypted/pkcs1-oaep + - k9WFg+z1sgabdmlWRa/6Cc37N2TxmzCGC23nNr/fC5eDf5z73D0XycpbMhvYZ4ZRqXuRD + Yl9kTaQ6yRiaUaQfHgqIR+8R5m/IrC+F57tageAiFljs1Yf895vA1TlpE0TCI8jUJs6M7 + z8f87FkrTZhgVfOI4KNUb+g9EioEs8/sx5WEXt053lm18xhnh7VyDE9DpVtoZ2swqpXTG + DPqlWLi2SHBU9PVTmwtz+mRA9BCNPFS4vUL1EQoMir9gqwYrEV3aezNPUeiWqMq7HqaQd + ZSGdOmxUTsjPZnrvGP6cAe9fEDAcqB9hoO1EKkLTirGN/9sEpK96vmMyIiTU91fMUCy/L + tp94a22CvOPEgL5E59M2HI0IEYLkhN9pjyRm608MoO9kcm7vt6rFstJafDp8Sfh4Klk3v + WGkZi4CnILpvpOXEdvG0cXb7HYJroBe1dblaFtS+AZWNYVo/PmA/gZ2DIifEGHsfHJhvp + 3cpIwBXY2BeP0lAtmHXf0afCP1aGNVlumKBz2zEhxT1hSxbmbjDuv7at/3kyI0KzuJQuU + fEDkHzF4U7XPW5FJudxqVew9wWxYap9DwWk5q6LdxI1/JKRyztNiUisk3o+I1LiKGfA0c + n+J3riTLmXl03zQXJRFNVdcexuolrtHNV5iTpN5P+/NT2SQXUj9Wmj44W7vz4g= + password: !encrypted/pkcs1-oaep + - ju6Q/Gr6Zp76tHJdag8Veejn2ovtTKtqXOa6UgCMN8kYdspOE0cUH9e6A80O2Sn3qwqiw + Y3cNWJQnK272Gmnm2r0tN8Y39l3G7n3dhbyimO4E/9NfMVeY/+GQEDSrlj+f7oxui99YS + gVfdA9L/fx4JCKRkqGOVT/IKMPTSRepcrS+R9o3kMFVtpz38G1CFfdnk2rTdLFVvgus5e + UY1Hougf37V/fJXDpf4NrCl//iKrJ0X8eafWkrVJ9H4x+Pni22zN54OjBMcle75fYQqxZ + VUFfKCedan9RvL6402NmoEY21WozoMmAPnoFe0I1XIiQecAaPs25snMrDJxX+prd4Lp6m + suyFlJcGHfvOaWIUmbBl2+OZwyZfl2LzCwYNR5OPbVZ8CjdJWWQ2e4Q8iRGbKqbA6cqUE + TXEnqmzobqfIR1Z9vS96kdAXEUo3MNcxyE600Vfwg0t4MyblYEXZjm+c7H8QCijM8Z8lX + 580awP/H504kfbqMInMuq2Ac+08b6NPObpHf2Q5hunrkpn6/7GS2fzhmkMPfRriNI5lOr + VCl4jWDJDYhtKh29q/ILrtmajHwyCxQjSjNqZKy31iWfinnQDYNFmgzdRxwvoYN3qwU1w + eYlzieYFM/znu+LAXaDE17O+25FYe9lk1s96Bm3jhGVARWize+sQSCo/xBor2A= diff --git a/Makefile b/Makefile index 21016bb2..0959f62a 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,14 @@ BUILD_CTX ?= src/bin DOCKER_REGISTRY ?= quay.io -IMAGE_PREFIX ?= attcomdev -IMAGE_TAG ?= latest +IMAGE_PREFIX ?= airshipit +IMAGE_TAG ?= untagged HELM ?= helm LABEL ?= commit-id IMAGE_NAME := airflow shipyard +PROXY ?= http://proxy.foo.com:8080 +USE_PROXY ?= false +PUSH_IMAGE ?= false IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG} IMAGE_DIR:=images/$(IMAGE_NAME) @@ -75,11 +78,25 @@ run: .PHONY: build_airflow build_airflow: - docker build -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile $(IMAGE_DIR) +ifeq ($(USE_PROXY), true) + docker build --network host -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile $(IMAGE_DIR) --build-arg http_proxy=$(PROXY) --build-arg https_proxy=$(PROXY) +else + docker build --network host -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile $(IMAGE_DIR) +endif +ifeq ($(PUSH_IMAGE), true) + docker push $(IMAGE) +endif .PHONY: build_shipyard build_shipyard: - docker build -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile --build-arg ctx_base=$(BUILD_CTX) . +ifeq ($(USE_PROXY), true) + docker build --network host -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile --build-arg ctx_base=$(BUILD_CTX) . --build-arg http_proxy=$(PROXY) --build-arg https_proxy=$(PROXY) +else + docker build --network host -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile --build-arg ctx_base=$(BUILD_CTX) . +endif +ifeq ($(PUSH_IMAGE), true) + docker push $(IMAGE) +endif .PHONY: clean clean: diff --git a/tools/gate/playbooks/osh-infra-build.yaml b/tools/gate/playbooks/osh-infra-build.yaml index d06296c1..1999000e 100644 --- a/tools/gate/playbooks/osh-infra-build.yaml +++ b/tools/gate/playbooks/osh-infra-build.yaml @@ -22,15 +22,3 @@ - build-helm-packages tags: - build-helm-packages - -- hosts: all - vars_files: - - vars.yaml - vars: - work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" - gather_facts: False - become: yes - roles: - - build-images - tags: - - build-images diff --git a/tools/gate/roles/build-images/tasks/airship-shipyard.yaml b/tools/gate/roles/build-images/tasks/airship-shipyard.yaml index cb8fb84f..a32cb62a 100644 --- a/tools/gate/roles/build-images/tasks/airship-shipyard.yaml +++ b/tools/gate/roles/build-images/tasks/airship-shipyard.yaml @@ -12,46 +12,70 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: set zuul_site_mirror_fqdn from env var if not defined - when: zuul_site_mirror_fqdn is not defined - set_fact: - zuul_site_mirror_fqdn: "{{ lookup('env','zuul_site_mirror_fqdn') }}" - -# NOTE(portdirect): Until https://github.com/ansible/ansible/issues/21433 is -# resolved, we build with a shell script to make use of the host network. -- name: Build Airship-Shipyard Image +- name: Debug tag generation inputs block: - # NOTE(portdirect): We do this to ensure we are feeding the docker build - # a clean path to work with. - - name: Airship-Shipyard image build path - shell: cd "{{ work_dir }}"; pwd - register: airship_shipyard_path - vars: - zuul_airship_shipyard_relative_path: ../airship-shipyard - work_dir: "{{ zuul.project.src_dir }}/{{ zuul_airship_shipyard_relative_path | default('') }}" + - debug: + var: publish + - debug: + var: tags + - debug: + var: zuul + - debug: + msg: "{{ tags | to_json }}" - - name: Airship-Shipyard image build path - shell: |- - set -ex; - docker build \ - --network host \ - --force-rm \ - --label zuul \ - -f images/shipyard/Dockerfile \ - --build-arg ctx_base=src/bin \ - {% if zuul_site_mirror_fqdn is defined and zuul_site_mirror_fqdn %} - --build-arg UBUNTU_URL="http://{{ zuul_site_mirror_fqdn }}/ubuntu/" \ - --build-arg ALLOW_UNAUTHENTICATED="true" \ - --build-arg PIP_INDEX_URL="http://{{ zuul_site_mirror_fqdn }}/pypi/simple" \ - --build-arg PIP_TRUSTED_HOST="{{ zuul_site_mirror_fqdn }}" \ - {% endif %} - . - args: - chdir: "{{ airship_shipyard_path.stdout }}" - executable: /bin/bash +- name: Determine tags + shell: echo '{{ tags | to_json }}' | python {{ zuul.project.src_dir }}/tools/image_tags.py + environment: + BRANCH: "{{ zuul.branch }}" + CHANGE: "{{ zuul.change }}" + COMMIT: "{{ zuul.newrev }}" + PATCHSET: "{{ zuul.patchset }}" + register: image_tags - - name: Register Shipyard Image ID - shell: |- - set -e; - echo $( docker images --filter label=zuul | awk '{print $3}' | head -2 | tail -1 ) - register: shipyard_image_id +- name: Debug computed tags + debug: + var: image_tags + +- name: Make images + when: not publish + block: + - make: + chdir: "{{ zuul.project.src_dir }}" + target: images + params: + IMAGE_TAG: "{{ item }}" + with_items: "{{ image_tags.stdout_lines }}" + + - shell: "docker images" + register: docker_images + + - debug: + var: docker_images + + become: True + +- name: Publish images + block: + - docker_login: + username: "{{ airship_shipyard_quay_creds.username }}" + password: "{{ airship_shipyard_quay_creds.password }}" + registry_url: "https://quay.io/api/v1/" + + - make: + chdir: "{{ zuul.project.src_dir }}" + target: images + params: + DOCKER_REGISTRY: "quay.io" + IMAGE_PREFIX: "airshipit" + IMAGE_TAG: "{{ item }}" + PUSH_IMAGE: "true" + with_items: "{{ image_tags.stdout_lines }}" + + - shell: "docker images" + register: docker_images + + - debug: + var: docker_images + + when: publish + become: True diff --git a/tools/image_tags.py b/tools/image_tags.py new file mode 100644 index 00000000..0b22fd48 --- /dev/null +++ b/tools/image_tags.py @@ -0,0 +1,126 @@ +#!/bin/python +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import sys + +LOG = logging.getLogger(__name__) + +LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(funcName)s [%(lineno)3d] %(message)s' # noqa + + +class TagGenExeception(Exception): + pass + + +def read_config(stream, env): + config = {} + try: + config['tags'] = json.load(stream) + except ValueError: + LOG.exception('Failed to decode JSON from input stream') + config['tags'] = {} + + LOG.debug('Configuration after reading stream: %s', config) + + config['context'] = { + 'branch': env.get('BRANCH'), + 'change': env.get('CHANGE'), + 'commit': env.get('COMMIT'), + 'ps': env.get('PATCHSET'), + } + + LOG.info('Final configuration: %s', config) + + return config + + +def build_tags(config): + tags = config.get('tags', {}).get('static', []) + LOG.debug('Dynamic tags: %s', tags) + tags.extend(build_dynamic_tags(config)) + LOG.info('All tags: %s', tags) + return tags + + +def build_dynamic_tags(config): + dynamic_tags = [] + + dynamic_tags.extend(_build_branch_tag(config)) + dynamic_tags.extend(_build_commit_tag(config)) + dynamic_tags.extend(_build_ps_tag(config)) + + return dynamic_tags + + +def _build_branch_tag(config): + if _valid_dg(config, 'branch'): + return [config['context']['branch']] + else: + return [] + + +def _build_commit_tag(config): + if _valid_dg(config, 'commit'): + return [config['context']['commit']] + else: + return [] + + +def _build_ps_tag(config): + if _valid_dg(config, 'patch_set', 'change') and _valid_dg( + config, 'patch_set', 'ps'): + return [ + '%s-%s' % (config['context']['change'], config['context']['ps']) + ] + else: + return [] + + +def _valid_dg(config, dynamic_tag, context_name=None): + if context_name is None: + context_name = dynamic_tag + + if config.get('tags', {}).get('dynamic', {}).get(dynamic_tag): + if config.get('context', {}).get(context_name): + return True + else: + raise TagGenExeception('Dynamic tag "%s" requested, but "%s"' + ' not found in context' % (dynamic_tag, + context_name)) + else: + return False + + +def main(): + config = read_config(sys.stdin, os.environ) + tags = build_tags(config) + + for tag in tags: + print(tag) + + +if __name__ == '__main__': + logging.basicConfig(format=LOG_FORMAT, level=logging.WARNING) + try: + main() + except TagGenExeception: + LOG.exception('Failed to generate tags') + sys.exit(1) + except: + LOG.exception('Unexpected exception') + sys.exit(2)