diff --git a/.zuul.yaml b/.zuul.yaml index 266ea4c3..06b177a0 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -26,27 +26,35 @@ - deckhand-tox-py35-postgresql - deckhand-functional-uwsgi-py35: voting: false - - deckhand-functional-docker-py35 + - deckhand-functional-docker-py35-ubuntu + - deckhand-functional-docker-py35-opensuse - deckhand-integration-uwsgi-py35: voting: false - - deckhand-integration-docker-py35: + - deckhand-integration-docker-py35-ubuntu: + voting: false + - deckhand-integration-docker-py35-opensuse: voting: false - deckhand-chart-build-gate - deckhand-chart-build-latest-htk - - deckhand-docker-build-gate + - deckhand-docker-build-gate-ubuntu + - deckhand-docker-build-gate-opensuse - deckhand-airskiff-deployment gate: jobs: - deckhand-tox-py27-postgresql - deckhand-tox-py35-postgresql - - deckhand-functional-docker-py35 + - deckhand-functional-docker-py35-ubuntu + - deckhand-functional-docker-py35-opensuse - deckhand-chart-build-gate - - deckhand-docker-build-gate + - deckhand-docker-build-gate-ubuntu + - deckhand-docker-build-gate-opensuse post: jobs: - - deckhand-docker-publish - - deckhand-docker-tag - deckhand-upload-git-mirror + - deckhand-docker-publish-ubuntu + - deckhand-docker-publish-opensuse + - deckhand-docker-tag-ubuntu + - deckhand-docker-tag-opensuse - nodeset: name: deckhand-single-node @@ -104,15 +112,37 @@ - openstack/openstack-helm-infra - job: - name: deckhand-functional-docker-py35 + name: deckhand-functional-docker-py35-ubuntu description: | Run tox-based functional tests for the Airship Deckhand project under cPython version 3.5. Uses tox with the ``functional-py35`` environment. + Ubuntu (default) image is built and used. parent: deckhand-functional-docker-base nodeset: deckhand-single-node vars: tox_envlist: functional disable_keystone: true + distro: ubuntu_xenial + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ + - ^etc/.*$ + - ^releasenotes/.*$ + - ^setup.cfg$ + - ^deckhand/tests/unit/.*$ + +- job: + name: deckhand-functional-docker-py35-opensuse + description: | + Run tox-based functional tests for the Airship Deckhand project under + cPython version 3.5. Uses tox with the ``functional-py35`` environment. + Opensuse image is built and used. + parent: deckhand-functional-docker-base + nodeset: deckhand-single-node + vars: + tox_envlist: functional + disable_keystone: true + distro: opensuse_15 irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -184,14 +214,15 @@ - ^releasenotes/.*$ - job: - name: deckhand-integration-docker-py35 + name: deckhand-integration-docker-py35-ubuntu description: | Run tox-based integration tests for the Airship Deckhand project under - cPython version 3.5. + cPython version 3.5. Builds ubuntu (default) deckhand image. parent: deckhand-integration-docker-base nodeset: openstack-helm-single-node vars: disable_keystone: false + distro: ubuntu_xenial - job: name: deckhand-airskiff-deployment @@ -219,7 +250,18 @@ - ^releasenotes/.*$ - job: - name: deckhand-docker-build-gate + name: deckhand-integration-docker-py35-opensuse + description: | + Run tox-based integration tests for the Airship Deckhand project under + cPython version 3.5. Builds opensuse deckhand image. + parent: deckhand-integration-docker-base + nodeset: openstack-helm-single-node + vars: + disable_keystone: false + distro: opensuse_15 + +- job: + name: deckhand-docker-build-gate-ubuntu timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml nodeset: deckhand-single-node @@ -232,15 +274,29 @@ - ^setup.cfg$ vars: publish: false + distro: ubuntu_xenial tags: dynamic: patch_set: true - job: - name: deckhand-docker-publish + name: deckhand-docker-build-gate-opensuse + timeout: 1800 + run: tools/gate/playbooks/docker-image-build.yaml + nodeset: deckhand-single-node + irrelevant-files: *non-code-files-template + vars: + publish: false + distro: opensuse_15 + tags: + dynamic: + patch_set: true + +- job: + name: deckhand-docker-publish-ubuntu description: | Runs on every merge, unless files in a dictionary below are changed. - Builds and publishes container images on quay.io with a set of tags + Builds and publishes container ubuntu images on quay.io with a set of tags listed in vars section. Waits in Zuul queue for a node (VM) assignment. timeout: 1800 run: tools/gate/playbooks/docker-image-build.yaml @@ -250,17 +306,41 @@ irrelevant-files: *non-code-files-template vars: publish: true + distro: ubuntu_xenial tags: dynamic: branch: true commit: true static: - latest + - job: - name: deckhand-docker-tag + name: deckhand-docker-publish-opensuse + description: | + Runs on every merge, unless files in a dictionary below are changed. + Builds and publishes container opensuse images on quay.io with a set of tags + listed in vars section. Waits in Zuul queue for a node (VM) assignment. + timeout: 1800 + run: tools/gate/playbooks/docker-image-build.yaml + nodeset: deckhand-single-node + secrets: + - deckhand_quay_creds + irrelevant-files: *non-code-files-template + vars: + publish: true + distro: opensuse_15 + tags: + dynamic: + branch: true + commit: true + static: + - latest + +- job: + name: deckhand-docker-tag-ubuntu description: | Runs on every merge when files in a dictionalry below are changed, and - adds git commit id tag onto the container image published on quay.io, + adds git commit id tag onto the ubuntu container image published on quay.io, which has `latest` tag set. Does not wait in queue for a node (VM) assignment, runs almost immediately. timeout: 1800 @@ -269,6 +349,24 @@ nodes: [] secrets: - deckhand_quay_creds + vars: + distro: ubuntu_xenial + +- job: + name: deckhand-docker-tag-opensuse + description: | + Runs on every merge when files in a dictionalry below are changed, and + adds git commit id tag onto the opensuse container image published on quay.io, + which has `latest` tag set. Does not wait in queue for a node (VM) + assignment, runs almost immediately. + timeout: 1800 + run: tools/gate/playbooks/docker-image-tag.yaml + nodeset: + nodes: [] + secrets: + - deckhand_quay_creds + vars: + distro: opensuse_15 # file pattern here must be exactly the same as in # deckhand-docker-publish job above, # job will be executed on merge only when any of this files get changed diff --git a/Makefile b/Makefile index 7489a6c4..aae560a5 100644 --- a/Makefile +++ b/Makefile @@ -12,20 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -BUILD_DIR := $(shell mkdir -p build && mktemp -d -p build) -DOCKER_REGISTRY ?= quay.io -IMAGE_NAME ?= deckhand -IMAGE_PREFIX ?= airshipit -IMAGE_TAG ?= latest -HELM := $(shell realpath $(BUILD_DIR))/helm -PROXY ?= http://proxy.foo.com:8000 -NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local -USE_PROXY ?= false -PUSH_IMAGE ?= false +BUILD_DIR := $(shell mkdir -p build && mktemp -d -p build) +DOCKER_REGISTRY ?= quay.io +IMAGE_NAME ?= deckhand +IMAGE_PREFIX ?= airshipit +IMAGE_TAG ?= latest +DISTRO_BASE_IMAGE ?= +HELM := $(shell realpath $(BUILD_DIR))/helm +PROXY ?= http://proxy.foo.com:8000 +NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local +USE_PROXY ?= false +PUSH_IMAGE ?= false # use this variable for image labels added in internal build process -LABEL ?= org.airshipit.build=community -COMMIT ?= $(shell git rev-parse HEAD) -IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG} +LABEL ?= org.airshipit.build=community +DISTRO ?= ubuntu_xenial +COMMIT ?= $(shell git rev-parse HEAD) +IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} export @@ -63,6 +65,8 @@ dry-run: clean tests: tox +_BASE_IMAGE_ARG := $(if $(DISTRO_BASE_IMAGE),--build-arg FROM="${DISTRO_BASE_IMAGE}" ,) + # Make targets intended for use by the primary targets above. .PHONY: build_deckhand build_deckhand: @@ -71,7 +75,8 @@ ifeq ($(USE_PROXY), true) --label "org.opencontainers.image.revision=$(COMMIT)" \ --label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \ --label "org.opencontainers.image.title=$(IMAGE_NAME)" \ - -f images/deckhand/Dockerfile \ + $(_BASE_IMAGE_ARG) \ + -f images/deckhand/Dockerfile.$(DISTRO) \ --build-arg http_proxy=$(PROXY) \ --build-arg https_proxy=$(PROXY) \ --build-arg HTTP_PROXY=$(PROXY) \ @@ -83,7 +88,8 @@ else --label "org.opencontainers.image.revision=$(COMMIT)" \ --label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \ --label "org.opencontainers.image.title=$(IMAGE_NAME)" \ - -f images/deckhand/Dockerfile . + $(_BASE_IMAGE_ARG) \ + -f images/deckhand/Dockerfile.$(DISTRO) . endif ifeq ($(PUSH_IMAGE), true) docker push $(IMAGE) diff --git a/doc/source/contributor/developer-overview.rst b/doc/source/contributor/developer-overview.rst index eba1bb3f..4d8b3941 100644 --- a/doc/source/contributor/developer-overview.rst +++ b/doc/source/contributor/developer-overview.rst @@ -110,9 +110,9 @@ See :ref:`coding-guide` for more information on contribution guidelines. Docker ------ -The Deckhand Dockerfile is located in ``/images/deckhand`` along with any -artifacts built specifically to enable the container image. Make targets are -used for generating and testing the artifacts. +The distribution specific Deckhand Dockerfile.{DISTRO} is located in ``/images/deckhand`` +along with any artifacts built specifically to enable the container image. Make +targets are used for generating and testing the artifacts. * ``make images`` - Build the Deckhand Docker image. diff --git a/doc/source/contributor/testing.rst b/doc/source/contributor/testing.rst index e154191b..925c9c7a 100644 --- a/doc/source/contributor/testing.rst +++ b/doc/source/contributor/testing.rst @@ -143,7 +143,7 @@ testing. To test Deckhand against a containerized image, run, for example: :: - export DECKHAND_IMAGE=quay.io/airshipit/deckhand:latest + export DECKHAND_IMAGE=quay.io/airshipit/deckhand:latest-ubuntu_xenial tox -e functional-dev Which will result in the following script output: @@ -152,7 +152,7 @@ Which will result in the following script output: Running Deckhand via Docker + sleep 5 - + sudo docker run --rm --net=host -p 9000:9000 -v /opt/stack/deckhand/tmp.oBJ6XScFgC:/etc/deckhand quay.io/airshipit/deckhand:latest + + sudo docker run --rm --net=host -p 9000:9000 -v /opt/stack/deckhand/tmp.oBJ6XScFgC:/etc/deckhand quay.io/airshipit/deckhand:latest-ubuntu_xenial .. warning:: diff --git a/doc/source/operators/index.rst b/doc/source/operators/index.rst index 63b27c03..57ac4395 100644 --- a/doc/source/operators/index.rst +++ b/doc/source/operators/index.rst @@ -24,6 +24,7 @@ Operator's Guide api_client configuration exceptions + multi-distro-support Indices and tables ------------------ diff --git a/doc/source/operators/multi-distro-support.rst b/doc/source/operators/multi-distro-support.rst new file mode 100644 index 00000000..55ff29c1 --- /dev/null +++ b/doc/source/operators/multi-distro-support.rst @@ -0,0 +1,92 @@ +.. + Copyright 2019 SUSE LLC + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + +Multiple Distro Support +======================= + +This project builds images for Deckhand component only. Currently, it +supports building images for ubuntu and opensuse ( leap 15.0 as base image). + +By default, Ubuntu images are built and are published to public registry +server. Recently support for publishing opensuse image has been added. + +If you need to build opensuse images locally, the following parameters +can be passed to the *make* command in deckhand repository's root +directory with *images* as target:: + + DISTRO: opensuse_15 + DISTRO_BASE_IMAGE: "opensuse/leap:15.0" + DOCKER_REGISTRY: { your_docker_registry } + IMAGE_TAG: latest + PUSH_IMAGE: false + +Following is an example in command format to build and publish images locally. +Command is run in deckhand repository's root directory. + + make images DISTRO=opensuse_15 DOCKER_REGISTRY={ your_docker_registry} \ + IMAGE_TAG=latest PUSH_IMAGE=true + + +Following parameters need to be passed as environment/shell variable to make +command: + +DISTRO + parameter to identify distro specific Dockerfile, ubuntu_xenial (Default) + +DISTRO_BASE_IMAGE + parameter to use different base image other than what's + used in DISTRO specific Dockerfile (optional) + +DOCKER_REGISTRY + parameter to specify local/internal docker registry if need + to publish image (optional), quay.io (Default) + +IMAGE_TAG + tag to be used for image built, untagged (Default) + +PUSH_IMAGE + flag to indicate if images needs to be pushed to a docker + registry, false (Default) + +This work is done as per approved spec `multi_distro_support`_. Currently only image +building logic is enhanced to support multiple distro. + + +Adding New Distro Support +-------------------------- + +To add support for building images for a new distro, following steps can be +followed. + + #. Distro specific deckhand image can be built and tested locally first. + + #. Add distro specific Dockerfile which will have steps to include necessary + packages and run environment configuration. Use existing Dockerfile as sample + to identify needed packages and environment information. + + #. New dockerfile can be named as Dockefile.{DISTRO} where DISTRO is expected to be + distro identifier which is passed to makefile. + + #. Respective dockerfile needs to be placed in {deckahnd_repo}/images/deckhand/ + + #. Add check, gate, and post jobs for building, testing and publishing images. These + entries need to be added in {deckhand_repo}/.zuul.yaml file. You may refer to + existing zuul file to opensuse support to understand its usage pattern. + + #. Add any relevant information to this document. + + +.. _multi_distro_support: https://airship-specs.readthedocs.io/en/latest/specs/approved/airship_multi_linux_distros.html diff --git a/doc/source/users/getting-started.rst b/doc/source/users/getting-started.rst index 3f182ae7..a51ada87 100644 --- a/doc/source/users/getting-started.rst +++ b/doc/source/users/getting-started.rst @@ -57,7 +57,7 @@ Finally, run Deckhand via Docker:: --net=host \ -p 9000:9000 \ -v $CONF_DIR:/etc/deckhand \ - quay.io/airshipit/deckhand:latest + quay.io/airshipit/deckhand:latest-ubuntu_xenial PostgreSQL ^^^^^^^^^^ @@ -101,7 +101,7 @@ Run an update to the Database to bring it to the current code level:: $ [sudo] docker run --rm \ --net=host \ -v $CONF_DIR:/etc/deckhand \ - quay.io/airshipit/deckhand:latest \ + quay.io/airshipit/deckhand:latest-ubuntu_xenial\ alembic upgrade head Finally, run Deckhand via Docker:: @@ -110,7 +110,7 @@ Finally, run Deckhand via Docker:: --net=host \ -p 9000:9000 \ -v $CONF_DIR:/etc/deckhand \ - quay.io/airshipit/deckhand:latest + quay.io/airshipit/deckhand:latest-ubuntu_xenial To kill the ephemeral DB afterward:: @@ -206,7 +206,7 @@ After, from the command line, execute: --net=host \ -p 9000:9000 \ -v $CONF_DIR:/etc/deckhand \ - quay.io/airshipit/deckhand:latest server + quay.io/airshipit/deckhand:latest-ubuntu_xenial server .. _development-utilities: diff --git a/images/deckhand/Dockerfile.opensuse_15 b/images/deckhand/Dockerfile.opensuse_15 new file mode 100644 index 00000000..e6ab8967 --- /dev/null +++ b/images/deckhand/Dockerfile.opensuse_15 @@ -0,0 +1,81 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG FROM=opensuse/leap:15.0 +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' +LABEL org.opencontainers.image.url='https://airshipit.org' +LABEL org.opencontainers.image.documentation='https://airship-deckhand.readthedocs.org' +LABEL org.opencontainers.image.source='https://opendev.org/airship/deckhand' +LABEL org.opencontainers.image.vendor='The Airship Authors' +LABEL org.opencontainers.image.licenses='Apache-2.0' + +ENV container docker +ENV PORT 9000 + +# Expose port 9000 for application +EXPOSE $PORT + +RUN set -x && \ + zypper -q update -y && \ + zypper install -y --no-recommends \ + git \ + curl \ + netcat-openbsd \ + netcfg \ + python3 \ + python3-setuptools \ + python3-pip \ + python3-devel \ + python3-python-dateutil \ + python3-dbm \ + ca-certificates \ + gcc \ + gcc-c++ \ + make \ + libffi-devel \ + libopenssl-devel \ + libpqxx-devel \ + && python3 -m pip install -U pip \ + && zypper clean -a \ + && rm -rf \ + /tmp/* \ + /var/tmp/* \ + /usr/share/man \ + /usr/share/doc \ + /usr/share/doc-base + +# Create deckhand user +RUN useradd -ms /bin/bash deckhand + +# Clone the deckhand repository +COPY . /home/deckhand/ + +# Change permissions +RUN chown -R deckhand: /home/deckhand \ + && chmod +x /home/deckhand/entrypoint.sh + +# Set work directory and install dependencies +WORKDIR /home/deckhand +RUN pip3 install -r requirements.txt +RUN python3 setup.py install + +# Set user to deckhand +USER deckhand + +# Execute entrypoint +ENTRYPOINT ["/home/deckhand/entrypoint.sh"] + +CMD ["server"] diff --git a/images/deckhand/Dockerfile b/images/deckhand/Dockerfile.ubuntu_xenial similarity index 95% rename from images/deckhand/Dockerfile rename to images/deckhand/Dockerfile.ubuntu_xenial index dbcbfa3f..9ff599f7 100644 --- a/images/deckhand/Dockerfile +++ b/images/deckhand/Dockerfile.ubuntu_xenial @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:16.04 +ARG FROM=ubuntu:16.04 +FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' LABEL org.opencontainers.image.url='https://airshipit.org' LABEL org.opencontainers.image.documentation='https://airship-deckhand.readthedocs.org' -LABEL org.opencontainers.image.source='https://git.openstack.org/openstack/airship-deckhand' +LABEL org.opencontainers.image.source='https://opendev.org/airship/deckhand' LABEL org.opencontainers.image.vendor='The Airship Authors' LABEL org.opencontainers.image.licenses='Apache-2.0' diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml index 114e7bee..3af05b57 100644 --- a/tools/gate/playbooks/docker-image-build.yaml +++ b/tools/gate/playbooks/docker-image-build.yaml @@ -18,6 +18,8 @@ block: - debug: var: publish + - debug: + var: distro - debug: var: tags - debug: @@ -59,6 +61,7 @@ target: images params: IMAGE_TAG: "{{ item }}" + DISTRO: "{{ distro }}" with_items: "{{ image_tags.stdout_lines }}" - shell: "docker images" @@ -85,6 +88,7 @@ IMAGE_TAG: "{{ item }}" COMMIT: "{{ zuul.newrev | default('') }}" PUSH_IMAGE: "true" + DISTRO: "{{ distro }}" with_items: "{{ image_tags.stdout_lines }}" - shell: "docker images" diff --git a/tools/gate/playbooks/docker-image-tag.yaml b/tools/gate/playbooks/docker-image-tag.yaml index cf0e456a..0cb8e289 100644 --- a/tools/gate/playbooks/docker-image-tag.yaml +++ b/tools/gate/playbooks/docker-image-tag.yaml @@ -16,6 +16,7 @@ - hosts: localhost vars: latest_tag: latest + distro: "{{ distro | default ('') }}" org: airshipit image: deckhand new_tag: "{{ zuul.newrev | default('') }}" @@ -27,9 +28,9 @@ var: zuul - block: - - name: Get id of the container image currently tagged as latest + - name: Get id of the distro specific container image currently tagged as latest uri: - url: "{{ quay_repo_api_url }}/{{ org }}/{{ image }}/tag/?specificTag={{ latest_tag }}&limit=1" + url: "{{ quay_repo_api_url }}/{{ org }}/{{ image }}/tag/?specificTag={{ latest_tag }}-{{ distro }}&limit=1" return_content: yes register: response_1 - name: Debug response diff --git a/tools/gate/roles/build-images/defaults/main.yaml b/tools/gate/roles/build-images/defaults/main.yaml index 29f96561..fa51aefb 100644 --- a/tools/gate/roles/build-images/defaults/main.yaml +++ b/tools/gate/roles/build-images/defaults/main.yaml @@ -16,3 +16,5 @@ proxy: http: null https: null noproxy: null + +distro: ubuntu_xenial diff --git a/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml b/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml index c12f2356..13e8ed00 100644 --- a/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml +++ b/tools/gate/roles/build-images/tasks/build-airship-deckhand-image.yaml @@ -37,7 +37,7 @@ --network host \ --force-rm \ --label zuul \ - --file images/deckhand/Dockerfile \ + --file images/deckhand/Dockerfile.{{ distro }} \ {% if zuul_site_mirror_fqdn is defined and zuul_site_mirror_fqdn %} --build-arg UBUNTU_URL="http://{{ zuul_site_mirror_fqdn }}/ubuntu/" \ --build-arg ALLOW_UNAUTHENTICATED="true" \ @@ -58,7 +58,7 @@ --network host \ --force-rm \ --label zuul \ - --file images/deckhand/Dockerfile \ + --file images/deckhand/Dockerfile.{{ distro }} \ --build-arg HTTP_PROXY="{{ proxy.http }}" \ --build-arg HTTPS_PROXY="{{ proxy.https }}" \ --build-arg NO_PROXY="{{ proxy.noproxy }}" \