Add lint and check gates for airship-utils

This is base CR for all other CRs in airship-utils.
It sets up gates config with .zuul.yaml.

It was intended that airship-utils project is a collection of utils
can be used in Airship. At the moment only one util is available -
miniMirror. So all gates checking aims to miniMirror checks.

Two gates available:

- Lint gate runs shellcheck on assets/*.sh
  and hadolint Dockerfile. Can be checked locally with "make lint".
- Check gate builds an image and runs container from the image.
  It also checks that it's possible to install Debian packages from
  the running container.

Change-Id: I354cf96a4ddcd9f1f077a4a93869178ec6c366ab

.zuul.yaml

@@ -0,0 +1,59 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- project:
+    check:
+      jobs:
+        - airship-utils-linter
+        - airship-utils-ubuntu
+
+    gate:
+      jobs:
+        - airship-utils-linter
+        - airship-utils-ubuntu
+
+- job:
+    name: airship-utils-linter
+    roles:
+      - zuul: openstack/openstack-helm-infra
+    vars:
+      zuul_osh_infra_relative_path: ../openstack-helm-infra/
+    timeout: 7200
+    pre-run:
+      - tools/gate/playbooks/osh-infra-upgrade-host.yaml
+      - tools/gate/playbooks/osh-infra-deploy-docker.yaml
+    run: tools/gate/playbooks/zuul-linter.yaml
+    post-run: tools/gate/playbooks/osh-infra-collect-logs.yaml
+    required-projects:
+      - openstack/openstack-helm-infra
+
+- job:
+    name: airship-utils
+    roles:
+      - zuul: openstack/openstack-helm-infra
+    vars:
+      zuul_osh_infra_relative_path: ../openstack-helm-infra/
+    timeout: 7200
+    pre-run:
+      - tools/gate/playbooks/osh-infra-upgrade-host.yaml
+      - tools/gate/playbooks/osh-infra-deploy-docker.yaml
+      - tools/gate/playbooks/osh-infra-build.yaml
+      - tools/gate/playbooks/osh-infra-deploy-k8s.yaml
+    run: tools/gate/playbooks/zuul-check.yaml
+    post-run: tools/gate/playbooks/osh-infra-collect-logs.yaml
+    required-projects:
+      - openstack/openstack-helm-infra
+
+- job:
+    name: airship-utils-ubuntu
+    parent: airship-utils
+    nodeset: openstack-helm-single-node

Makefile

@@ -14,13 +14,13 @@
 
 IMAGE_PREFIX               ?= airshipit
 IMAGE_TAG                  ?= untagged
-IMAGE_NAME                 := aptly
+IMAGE_NAME                 := mini-mirror
 COMMIT                     ?= commit-id
 
 DOCKER_REGISTRY            ?= quay.io
 PUSH_IMAGE                 ?= false
 
-HELM                       := $(BUILD_DIR)/helm
+HELM                       := helm
 
 PROXY                      ?= http://proxy.foo.com:8000
 NO_PROXY                   ?= localhost,127.0.0.1,.svc.cluster.local
@@ -30,11 +30,16 @@ UBUNTU_BASE_IMAGE          ?= ubuntu:16.04
 
 IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}
 
-.PHONY: validate
-validate: lint tests
+MINI_MIRROR                := mini-mirror
 
-.PHONY: tests
-tests: clean build
+.PHONY: validate
+validate: lint test
+
+.PHONY: test
+test: test-containers
+
+.PHONY: test-containers
+test-containers: clean build
 	docker run -d \
 		--publish 8080:80 \
 		--volume $(shell pwd)/assets/nginx:/opt/nginx \
@@ -52,13 +57,13 @@ clean:
 
 .PHONY: lint
 lint:
-	shellcheck assets/*.sh
-	hadolint Dockerfile
+	shellcheck $(MINI_MIRROR)/assets/*.sh
+	docker run --rm -i hadolint/hadolint <  $(MINI_MIRROR)/Dockerfile
 
 .PHONY: build
 build:
 ifeq ($(USE_PROXY), true)
-	docker build --network host -t $(IMAGE) \
+	cd $(MINI_MIRROR); docker build --network host -t $(IMAGE) \
 		--label "org.opencontainers.image.revision=$(COMMIT)" \
 		--label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \
 		--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
@@ -70,7 +75,7 @@ ifeq ($(USE_PROXY), true)
 		--build-arg no_proxy=$(NO_PROXY) \
 		--build-arg NO_PROXY=$(NO_PROXY) .
 else
-	docker build --network host -t $(IMAGE) \
+	cd $(MINI_MIRROR); docker build --network host -t $(IMAGE) \
 		--label "org.opencontainers.image.revision=$(COMMIT)" \
 		--label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \
 		--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
@@ -79,3 +84,7 @@ endif
 ifeq ($(PUSH_IMAGE), true)
 	docker push $(IMAGE)
 endif
+
+.PHONY: lint-install
+lint-install:
+	apt-get install -y shellcheck

mini-mirror/Dockerfile

@@ -32,10 +32,10 @@ RUN apt-get -q update \
     gpgv=1.4.20-1ubuntu3.3 \
     graphviz=2.38.0-12ubuntu2.1 \
     supervisor=3.2.0-2ubuntu0.2 \
-    nginx=1.10.3-0ubuntu0.16.04.2 \
+    nginx=1.10.3-0ubuntu0.16.04.3 \
     wget=1.17.1-1ubuntu1.4 \
     xz-utils=5.1.1alpha+20120614-2ubuntu2 \
-    apt-utils=1.2.27 \
+    apt-utils=1.2.29 \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 

mini-mirror/assets/packages/my_packages

@@ -0,0 +1,5 @@
+mysql-client-5.7 (= 5.7.24-0ubuntu0.16.04.1)
+mysql-client-core-5.7
+postgresql-client-9.5 (= 9.5.14-0ubuntu0.16.04)
+postgresql-client-common
+telnet

tools/gate/playbooks/osh-infra-build.yaml

@@ -0,0 +1,36 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: True
+  roles:
+    - build-helm-packages
+  tags:
+    - build-helm-packages
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: False
+  become: yes
+  roles:
+    - build-images
+  tags:
+    - build-images

tools/gate/playbooks/osh-infra-collect-logs.yaml

@@ -0,0 +1,30 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+    logs_dir: "/tmp/logs"
+  roles:
+    - gather-host-logs
+    - helm-release-status
+    - describe-kubernetes-objects
+    - gather-pod-logs
+    - gather-prom-metrics
+  tags:
+    - gather-host-logs
+    - helm-release-status
+    - describe-kubernetes-objects
+    - gather-pod-logs
+    - gather-prom-metrics

tools/gate/playbooks/osh-infra-deploy-docker.yaml

@@ -0,0 +1,43 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: False
+  become: yes
+  roles:
+    - deploy-python
+  tags:
+    - deploy-python
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: True
+  become: yes
+  roles:
+    - setup-firewall
+    - deploy-python-pip
+    - deploy-docker
+    - deploy-jq
+  tags:
+    - setup-firewall
+    - deploy-python-pip
+    - deploy-docker
+    - deploy-jq

tools/gate/playbooks/osh-infra-deploy-k8s.yaml

@@ -0,0 +1,36 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: True
+  roles:
+    - build-helm-packages
+    - deploy-kubeadm-aio-master
+  tags:
+    - build-helm-packages
+    - deploy-kubeadm-aio-master
+
+- hosts: nodes
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  roles:
+    - deploy-kubeadm-aio-node
+  tags:
+    - deploy-kubeadm-aio-node

tools/gate/playbooks/osh-infra-upgrade-host.yaml

@@ -0,0 +1,41 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: False
+  become: yes
+  roles:
+    - deploy-python
+  tags:
+    - deploy-python
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: True
+  become: yes
+  roles:
+    - upgrade-host
+    - start-zuul-console
+    - disable-local-nameserver
+  tags:
+    - upgrade-host
+    - start-zuul-console
+    - disable-local-nameserver

tools/gate/playbooks/vars.yaml

@@ -0,0 +1,15 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+null: null

tools/gate/playbooks/zuul-check.yaml

@@ -0,0 +1,21 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Build and Run Container
+      become: true
+      command: make test
+      args:
+        chdir: "{{ zuul_osh_relative_path | default(zuul.project.src_dir) }}"

tools/gate/playbooks/zuul-linter.yaml

@@ -0,0 +1,21 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Execute a Linter check
+      become: true
+      command: make lint-install lint
+      args:
+        chdir: "{{ zuul_osh_relative_path | default(zuul.project.src_dir) }}"