From b9f2ce32b3b1d33a66ccab07cdb7f1d2fee41207 Mon Sep 17 00:00:00 2001 From: Alexander Hughes Date: Tue, 7 May 2019 12:11:24 -0500 Subject: [PATCH] Add docker build gate There is a future requirement for Spyglass to have helm charts. These charts will need docker images published in a source countrolled manner. This change depends on https://review.opendev.org/#/c/657614/ which adds the airship_spyglass_quay_creds parameters. This patch: 1. Adds the docker-image-build playbook for Spyglass. 2. Uses the playbook in zuul jobs: check, gate and post. 3. Adds the spyglass-single-node nodeset. 4. Adds the vars.yaml file required by docker-image-build playbook. 5. Adds the image_tags.py script required by docker-image-build playbook. 6. Adds the docker-systemd.conf file required by docker-image-build playbook. Change-Id: I76981e719a7e42984089fec08e1161a6923c5c52 --- .zuul.yaml | 38 +++++ tools/gate/playbooks/docker-image-build.yaml | 130 ++++++++++++++++++ .../gate/playbooks/files/docker-systemd.conf | 22 +++ tools/gate/playbooks/vars.yaml | 19 +++ tools/image_tags.py | 127 +++++++++++++++++ 5 files changed, 336 insertions(+) create mode 100644 tools/gate/playbooks/docker-image-build.yaml create mode 100644 tools/gate/playbooks/files/docker-systemd.conf create mode 100644 tools/gate/playbooks/vars.yaml create mode 100644 tools/image_tags.py diff --git a/.zuul.yaml b/.zuul.yaml index 572324d..bf57a69 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -13,9 +13,47 @@ check: jobs: - openstack-tox-pep8 + - spyglass-docker-build-gate gate: jobs: - openstack-tox-pep8 + - spyglass-docker-build-gate + post: + jobs: + - spyglass-docker-publish + +- nodeset: + name: spyglass-single-node + nodes: + - name: primary + label: ubuntu-xenial + +- job: + name: spyglass-docker-build-gate + timeout: 1800 + run: tools/gate/playbooks/docker-image-build.yaml + nodeset: spyglass-single-node + vars: + publish: false + tags: + dynamic: + patch_set: true + +- job: + name: spyglass-docker-publish + timeout: 1800 + run: tools/gate/playbooks/docker-image-build.yaml + nodeset: armada-single-node + secrets: + - airship_spyglass_quay_creds + vars: + publish: true + tags: + dynamic: + branch: true + commit: true + static: + - latest - secret: name: airship_spyglass_quay_creds diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml new file mode 100644 index 0000000..39e222b --- /dev/null +++ b/tools/gate/playbooks/docker-image-build.yaml @@ -0,0 +1,130 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + tasks: + - include_vars: vars.yaml + + - name: Debug tag generation inputs + block: + - debug: + var: publish + - debug: + var: tags + - debug: + var: zuul + - debug: + msg: "{{ tags | to_json }}" + + - name: Determine tags + shell: echo '{{ tags | to_json }}' | python3 {{ zuul.project.src_dir }}/tools/image_tags.py + environment: + BRANCH: "{{ zuul.branch | default('') }}" + CHANGE: "{{ zuul.change | default('') }}" + COMMIT: "{{ zuul.newrev | default('') }}" + PATCHSET: "{{ zuul.patchset | default('') }}" + register: image_tags + + - name: Debug computed tags + debug: + var: image_tags + + - name: Install Docker (Debian) + when: ansible_os_family == 'Debian' + block: + - file: + path: "{{ item }}" + state: directory + with_items: + - /etc/docker/ + - /etc/systemd/system/docker.service.d/ + - /var/lib/docker/ + - mount: + path: /var/lib/docker/ + src: tmpfs + fstype: tmpfs + opts: size=25g + state: mounted + - copy: "{{ item }}" + with_items: + - content: "{{ docker_daemon | to_json }}" + dest: /etc/docker/daemon.json + - src: files/docker-systemd.conf + dest: /etc/systemd/system/docker.service.d/ + - apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + - apt_repository: + repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker xenial stable + - apt: + name: "{{ item }}" + allow_unauthenticated: True + with_items: + - docker-ce + - python-pip + - pip: + name: docker + version: 2.7.0 + # NOTE(SamYaple): Allow all connections from containers to host so the + # containers can access the http server for git and wheels + - iptables: + action: insert + chain: INPUT + in_interface: docker0 + jump: ACCEPT + become: True + + - name: Make images + when: not publish + block: + - make: + chdir: "{{ zuul.project.src_dir }}" + target: images + params: + IMAGE_TAG: "{{ item }}" + with_items: "{{ image_tags.stdout_lines }}" + + - shell: "docker images" + register: docker_images + + - debug: + var: docker_images + + become: True + + - name: Publish images + block: + - docker_login: + username: "{{ airship_spyglass_quay_creds.username }}" + password: "{{ airship_spyglass_quay_creds.password }}" + registry_url: "https://quay.io/api/v1/" + + - make: + chdir: "{{ zuul.project.src_dir }}" + target: images + params: + DOCKER_REGISTRY: "quay.io" + IMAGE_PREFIX: "airshipit" + IMAGE_TAG: "{{ item }}" + COMMIT: "{{ zuul.newrev | default('') }}" + PUSH_IMAGE: "true" + with_items: "{{ image_tags.stdout_lines }}" + + - shell: "docker images" + register: docker_images + + - debug: + var: docker_images + + when: publish + become: True diff --git a/tools/gate/playbooks/files/docker-systemd.conf b/tools/gate/playbooks/files/docker-systemd.conf new file mode 100644 index 0000000..6f2ca66 --- /dev/null +++ b/tools/gate/playbooks/files/docker-systemd.conf @@ -0,0 +1,22 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# NOTE(SamYaple): CentOS cannot be build with userns-remap enabled. httpd uses +# cap_set_file capability and there is no way to pass that in at build as of +# docker 17.06. +# TODO(SamYaple): Periodically check to see if this is possible in newer +# versions of Docker +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd diff --git a/tools/gate/playbooks/vars.yaml b/tools/gate/playbooks/vars.yaml new file mode 100644 index 0000000..0ccd26c --- /dev/null +++ b/tools/gate/playbooks/vars.yaml @@ -0,0 +1,19 @@ +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker_daemon: + group: zuul + registry-mirrors: + - "http://{{ zuul_site_mirror_fqdn }}:8082/" + storage-driver: overlay2 diff --git a/tools/image_tags.py b/tools/image_tags.py new file mode 100644 index 0000000..2a53aa7 --- /dev/null +++ b/tools/image_tags.py @@ -0,0 +1,127 @@ +#!/usr/bin/python3 +# Copyright 2019 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import sys + +LOG = logging.getLogger(__name__) + +LOG_FORMAT = '%(asctime)s %(levelname)-8s %(name)s:%(filename)s:' \ + '%(lineno)3d:%(funcName)s %(message)s' + + +class TagGenExeception(Exception): + pass + + +def read_config(stream, env): + config = {} + try: + config['tags'] = json.load(stream) + except ValueError: + LOG.exception('Failed to decode JSON from input stream') + config['tags'] = {} + + LOG.debug('Configuration after reading stream: %s', config) + + config['context'] = { + 'branch': env.get('BRANCH'), + 'change': env.get('CHANGE'), + 'commit': env.get('COMMIT'), + 'ps': env.get('PATCHSET'), + } + + LOG.info('Final configuration: %s', config) + + return config + + +def build_tags(config): + tags = config.get('tags', {}).get('static', []) + LOG.debug('Dynamic tags: %s', tags) + tags.extend(build_dynamic_tags(config)) + LOG.info('All tags: %s', tags) + return tags + + +def build_dynamic_tags(config): + dynamic_tags = [] + + dynamic_tags.extend(_build_branch_tag(config)) + dynamic_tags.extend(_build_commit_tag(config)) + dynamic_tags.extend(_build_ps_tag(config)) + + return dynamic_tags + + +def _build_branch_tag(config): + if _valid_dg(config, 'branch'): + return [config['context']['branch']] + else: + return [] + + +def _build_commit_tag(config): + if _valid_dg(config, 'commit'): + return [config['context']['commit']] + else: + return [] + + +def _build_ps_tag(config): + if _valid_dg(config, 'patch_set', 'change') and _valid_dg( + config, 'patch_set', 'ps'): + return [ + '%s-%s' % (config['context']['change'], config['context']['ps']) + ] + else: + return [] + + +def _valid_dg(config, dynamic_tag, context_name=None): + if context_name is None: + context_name = dynamic_tag + + if config.get('tags', {}).get('dynamic', {}).get(dynamic_tag): + if config.get('context', {}).get(context_name): + return True + else: + raise TagGenExeception('Dynamic tag "%s" requested, but "%s"' + ' not found in context' % (dynamic_tag, + context_name)) + else: + return False + + +def main(): + config = read_config(sys.stdin, os.environ) + tags = build_tags(config) + + for tag in tags: + print(tag) + + +if __name__ == '__main__': + logging.basicConfig(format=LOG_FORMAT, level=logging.WARNING) + try: + main() + except TagGenExeception: + LOG.exception('Failed to generate tags') + sys.exit(1) + except Exception: + LOG.exception('Unexpected exception') + sys.exit(2)