diff --git a/.zuul.yaml b/.zuul.yaml index b330506c..c94d83a1 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -13,23 +13,33 @@ - project: templates: - docs-on-readthedocs + - openstack-python38-jobs + - openstack-python36-jobs vars: rtd_webhook_id: '38809' rtd_project_name: 'airship-drydock' check: jobs: - - airship-drydock-omni-test + - airship-drydock-openstack-tox-cover-focal + - airship-drydock-openstack-tox-pep8-focal + - airship-drydock-baclient-test - airship-drydock-chart-build-gate - airship-drydock-chart-build-latest-htk - airship-drydock-docker-build-gate-ubuntu_focal + - airship-drydock-docker-build-gate-ubuntu_bionic + gate: jobs: - - airship-drydock-omni-test + - airship-drydock-openstack-tox-cover-focal + - airship-drydock-openstack-tox-pep8-focal + - airship-drydock-baclient-test - airship-drydock-chart-build-gate - airship-drydock-docker-build-gate-ubuntu_focal + - airship-drydock-docker-build-gate-ubuntu_bionic post: jobs: - airship-drydock-docker-publish-ubuntu_focal + - airship-drydock-docker-publish-ubuntu_bionic - drydock-upload-git-mirror - nodeset: @@ -45,13 +55,24 @@ label: ubuntu-focal - job: - name: airship-drydock-omni-test + name: airship-drydock-openstack-tox-cover-focal + parent: openstack-tox-cover + description: Runs cover job on focal + nodeset: airship-drydock-single-node-focal + pre-run: tools/gate/playbooks/install-docker.yaml + + +- job: + name: airship-drydock-openstack-tox-pep8-focal + parent: openstack-tox-pep8 + description: Runs pep8 job on focal + nodeset: airship-drydock-single-node-focal + +- job: + name: airship-drydock-baclient-test description: | - Run a set of lightweight lints and tests - (pep8, Helm chart lint, Sphinx build, Python unit tests, Bandit scan) - run: tools/gate/playbooks/omni_test.yaml - required-projects: - - openstack/openstack-helm-infra + Run a bootaction drydock client test + run: tools/gate/playbooks/baclient_test.yaml timeout: 3600 nodeset: airship-drydock-single-node-focal @@ -62,6 +83,8 @@ timeout: 900 run: tools/gate/playbooks/build-charts.yaml nodeset: airship-drydock-single-node-focal + vars: + HTK_COMMIT: a7cd689280cdbc0acd04a7a1b745941260e8700b - job: name: airship-drydock-chart-build-latest-htk @@ -90,6 +113,22 @@ patch_set: true +- job: + name: airship-drydock-docker-build-gate-ubuntu_bionic + timeout: 1800 + run: tools/gate/playbooks/docker-image-build.yaml + nodeset: airship-drydock-single-node + irrelevant-files: + - '^doc/.*' + - '^charts/.*' + vars: + publish: false + distro: ubuntu_bionic + tags: + dynamic: + patch_set: true + + - job: name: airship-drydock-docker-publish-ubuntu_focal timeout: 1800 @@ -109,6 +148,28 @@ commit: true static: - latest + +- job: + name: airship-drydock-docker-publish-ubuntu_bionic + timeout: 1800 + run: tools/gate/playbooks/docker-image-build.yaml + nodeset: airship-drydock-single-node-focal + secrets: + - airship_drydock_quay_creds + irrelevant-files: + - '^doc/.*' + - '^charts/.*' + vars: + publish: true + distro: ubuntu_bionic + tags: + dynamic: + branch: true + commit: true + static: + - latest + + - secret: name: airship_drydock_quay_creds data: @@ -143,15 +204,15 @@ git_mirror_repository: airshipit/drydock secrets: - name: git_mirror_credentials - secret: drydock-airshipit-github-secret + secret: drydock-airshipit-github-secret-2023-03-23 pass-to-parent: true - secret: - name: drydock-airshipit-github-secret + name: drydock-airshipit-github-secret-2023-03-23 data: user: git host: github.com - host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== + host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= ssh_key: !encrypted/pkcs1-oaep - UjLtbrpoJqWLQQBAjOWgiHPpJP6WqYenCJYtkv10ayR0esOS0BOUVSOCUC2TC+fBKx/rn dREF8XuQSDH2gmSUvTafzMhGZmy+Rk/KOi2Et+V3uNCdczxFPEovliQCQmt2LwZADubMg diff --git a/Makefile b/Makefile index ace042ed..82d3d80c 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ BUILD_DIR := $(shell mkdir -p build && mktemp -d -p build) DOCKER_REGISTRY ?= quay.io IMAGE_NAME ?= drydock IMAGE_PREFIX ?= airshipit -IMAGE_TAG ?= dev +IMAGE_TAG ?= latest HELM := $(shell realpath $(BUILD_DIR))/helm UBUNTU_BASE_IMAGE ?= DISTRO ?= ubuntu_focal @@ -35,7 +35,7 @@ export images: build_drydock # Run an image locally and exercise simple tests -run_images: run_drydock +run_images: build_drydock run_drydock # Run tests tests: pep8 security docs unit_tests test_baclient @@ -57,7 +57,7 @@ req_freeze: tox -re freeze # Run the drydock container and exercise simple tests -run_drydock: build_drydock +run_drydock: tools/drydock_image_run.sh # It seems CICD expects the target 'drydock' to diff --git a/bindep.txt b/bindep.txt index 0412a13d..85c6a48c 100644 --- a/bindep.txt +++ b/bindep.txt @@ -18,7 +18,7 @@ python3-dev python3-pip python3-setuptools ssh -tox + # PlantUML is used for documentation builds, graphviz is it's soft dependancy plantuml diff --git a/images/drydock/Dockerfile.ubuntu_bionic b/images/drydock/Dockerfile.ubuntu_bionic index b17c9340..8943ecd6 100644 --- a/images/drydock/Dockerfile.ubuntu_bionic +++ b/images/drydock/Dockerfile.ubuntu_bionic @@ -23,13 +23,18 @@ ENV container docker ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 -# Copy direct dependency requirements only to build a dependency layer -RUN echo "deb ${UBUNTU_REPO} bionic main restricted universe multiverse" > /etc/apt/sources.list; \ - echo "deb ${UBUNTU_REPO} bionic-security main restricted universe multiverse" >> /etc/apt/sources.list; \ - echo "deb ${UBUNTU_REPO} bionic-updates main restricted universe multiverse" >> /etc/apt/sources.list; \ - cat /etc/apt/sources.list; \ - echo "APT::Get::AllowUnauthenticated ${ALLOW_UNAUTHENTICATED};" >> /etc/apt/apt.conf.d/00-local-mirrors; - +COPY ./bindep.txt /tmp/drydock/ +WORKDIR /tmp/drydock +RUN apt update \ + && apt install -y --allow-downgrades \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + python3-setuptools \ + && DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y tzdata \ + && pip3 install bindep \ + && bindep -f /tmp/drydock/bindep.txt --brief | xargs apt install -y COPY ./tools/baclient_build.sh /tmp/drydock/ COPY ./go /tmp/drydock/go @@ -73,29 +78,36 @@ ENV PORT 9000 ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 -# Copy direct dependency requirements only to build a dependency layer -RUN echo "deb ${UBUNTU_REPO} bionic main restricted universe multiverse" > /etc/apt/sources.list; \ - echo "deb ${UBUNTU_REPO} bionic-security main restricted universe multiverse" >> /etc/apt/sources.list; \ - echo "deb ${UBUNTU_REPO} bionic-updates main restricted universe multiverse" >> /etc/apt/sources.list; \ - cat /etc/apt/sources.list; \ - echo "APT::Get::AllowUnauthenticated ${ALLOW_UNAUTHENTICATED};" >> /etc/apt/apt.conf.d/00-local-mirrors; - - -COPY ./requirements-host.txt /tmp/drydock/ -COPY ./hostdeps.sh /tmp/drydock -WORKDIR /tmp/drydock -RUN ./hostdeps.sh; \ - rm -r /var/lib/apt/lists/* +RUN DEBIAN_FRONTEND=noninteractive \ + apt update \ + && DEBIAN_FRONTEND=noninteractive \ + apt install -y \ + --allow-downgrades \ + --no-install-recommends \ + python3-dev \ + python3-pip \ + python3-setuptools \ + pkg-config \ + libvirt-dev \ + libssl-dev \ + libpq-dev \ + gcc \ + ssh\ + curl \ + netbase \ + && rm -r /var/lib/apt/lists/* # Install LibYAML ENV LD_LIBRARY_PATH=/usr/local/lib COPY --from=baclient_builder /usr/local/lib /usr/local/lib COPY --from=baclient_builder /usr/local/include/yaml.h /usr/local/include/yaml.h -COPY ./python/requirements-frozen.txt /tmp/drydock/ -RUN pip3 install \ +COPY ./python/requirements-frozen-bionic.txt /tmp/drydock/ +RUN \ + pip3 install --upgrade pip \ + && pip3 install \ --no-cache-dir \ - -r /tmp/drydock/requirements-frozen.txt + -r /tmp/drydock/requirements-frozen-bionic.txt COPY ./python /tmp/drydock/python WORKDIR /tmp/drydock/python diff --git a/images/drydock/Dockerfile.ubuntu_focal b/images/drydock/Dockerfile.ubuntu_focal index 933a2a75..5ed834dc 100644 --- a/images/drydock/Dockerfile.ubuntu_focal +++ b/images/drydock/Dockerfile.ubuntu_focal @@ -95,6 +95,7 @@ RUN DEBIAN_FRONTEND=noninteractive \ pkg-config \ libvirt-dev \ libssl-dev \ + libpq-dev \ gcc \ ssh\ curl \ @@ -107,7 +108,9 @@ COPY --from=baclient_builder /usr/local/lib /usr/local/lib COPY --from=baclient_builder /usr/local/include/yaml.h /usr/local/include/yaml.h COPY ./python/requirements-frozen.txt /tmp/drydock/ -RUN pip3 install \ +RUN \ + pip3 install --upgrade pip \ + && pip3 install \ --no-cache-dir \ -r /tmp/drydock/requirements-frozen.txt diff --git a/python/requirements-frozen-bionic.txt b/python/requirements-frozen-bionic.txt new file mode 100644 index 00000000..382b98cc --- /dev/null +++ b/python/requirements-frozen-bionic.txt @@ -0,0 +1,99 @@ +alembic==1.7.7 +amqp==5.1.1 +attrs==22.2.0 +bcrypt==4.0.1 +Beaker==1.12.1 +cached-property==1.5.2 +cachetools==4.2.4 +certifi==2023.5.7 +cffi==1.15.1 +charset-normalizer==2.0.12 +click==8.0.4 +cryptography==40.0.2 +debtcollector==2.5.0 +decorator==5.1.1 +defusedxml==0.7.1 +dnspython==2.2.1 +dogpile.cache==1.2.1 +eventlet==0.33.3 +falcon==3.1.1 +fasteners==0.18 +fixtures==4.0.1 +futurist==2.4.1 +greenlet==2.0.2 +idna==3.4 +importlib-metadata==4.8.3 +importlib-resources==5.4.0 +iso8601==1.1.0 +Jinja2==3.0.3 +jsonpatch==1.32 +jsonpath-rw==1.4.0 +jsonpointer==2.3 +jsonschema==3.2.0 +keystoneauth1==5.1.2 +keystonemiddleware==9.5.0 +kombu==5.1.0 +libvirt-python==9.3.0 +Mako==1.1.6 +MarkupSafe==2.0.1 +msgpack==1.0.5 +netaddr==0.8.0 +netifaces==0.11.0 +oauthlib==3.2.2 +os-service-types==1.7.0 +oslo.cache==2.11.0 +oslo.concurrency==4.5.1 +oslo.config==8.8.0 +oslo.context==4.1.0 +oslo.i18n==5.1.0 +oslo.log==4.8.0 +oslo.messaging==12.14.0 +oslo.metrics==0.4.0 +oslo.middleware==4.5.1 +oslo.policy==3.12.1 +oslo.serialization==4.3.0 +oslo.service==2.8.0 +oslo.utils==4.13.0 +oslo.versionedobjects==2.6.0 +packaging==21.3 +Paste==3.5.3 +PasteDeploy==2.1.1 +pbr==5.11.1 +pip==21.3.1 +ply==3.11 +prometheus-client==0.16.0 +psycopg2-binary==2.9.6 +PTable==0.9.2 +pycadf==3.1.1 +pycparser==2.21 +pyghmi==1.5.61 +pylibyaml==0.1.0 +pymongo==4.1.1 +pyparsing==3.0.9 +pyrsistent==0.18.0 +python-dateutil==2.8.2 +python-keystoneclient==4.5.0 +pytz==2023.3 +PyYAML==6.0 +redfish==3.1.9 +repoze.lru==0.7 +requests==2.27.1 +requests-toolbelt==1.0.0 +requests-unixsocket==0.3.0 +rfc3986==1.5.0 +Routes==2.5.1 +setuptools==59.6.0 +six==1.16.0 +SQLAlchemy==1.3.20 +statsd==4.0.1 +stevedore==3.5.2 +typing_extensions==4.1.1 +ulid2==0.3.0 +urllib3==1.26.16 +uWSGI==2.0.21 +vine==5.0.0 +WebOb==1.8.7 +wheel==0.37.1 +wrapt==1.15.0 +yappi==1.4.0 +zipp==3.6.0 diff --git a/python/test-requirements.txt b/python/test-requirements.txt index 0ea77b9e..92e9d790 100644 --- a/python/test-requirements.txt +++ b/python/test-requirements.txt @@ -1,19 +1,18 @@ # tests -click==6.7 +click falcon==3.1.1 jsonschema==3.2.0 mock==5.0.1 pylibyaml==0.1.0 -pymongo==3.10.1 +pymongo pytest >= 3.0 pytest-cov==4.0.0 pytest-mock -responses==0.23.1 -setuptools==56.0.0 -ulid2==0.1.1 +responses +ulid2 # tools yapf flake8 -bandit>=1.1.0 -sphinx>=1.6.2 +bandit==1.6.0 +sphinx<7.0.0 diff --git a/python/tests/postgres/start_postgres.sh b/python/tests/postgres/start_postgres.sh index 3dfaeef7..90443883 100755 --- a/python/tests/postgres/start_postgres.sh +++ b/python/tests/postgres/start_postgres.sh @@ -1,21 +1,28 @@ #!/bin/bash set -x + +DOCKER_REGISTRY=${DOCKER_REGISTRY:-"quay.io"} +IMAGE_PREFIX=${IMAGE_PREFIX:-"airshipit"} +IMAGE_NAME=${IMAGE_NAME:-"drydock"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +DISTRO=${DISTRO:-"ubuntu_focal"} + + IMAGE="${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}" -env if [[ ! -z $(docker ps | grep 'psql_integration') ]] then - sudo docker stop 'psql_integration' + docker stop 'psql_integration' fi -IMAGE=${IMAGE:-"quay.io/airshipit/drydock:latest-ubuntu_focal"} + if [[ ! -z $(docker ps | grep 'psql_integration') ]] then - sudo docker stop 'psql_integration' + docker stop 'psql_integration' fi -sudo docker run --rm -d -p 5432:5432/tcp -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_PASSWORD=postgres --name 'psql_integration' postgres:14.6 +docker run --rm -dp 5432:5432 --name 'psql_integration' -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6 sleep 15 docker run --rm --net host postgres:14.6 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres @@ -23,4 +30,4 @@ docker run --rm --net host postgres:14.6 psql -h localhost -c "create database d export DRYDOCK_DB_URL="postgresql+psycopg2://drydock:drydock@localhost:5432/drydock" -sudo docker run --rm -t --net=host -e DRYDOCK_DB_URL="$DRYDOCK_DB_URL" -w /tmp/drydock --entrypoint /usr/local/bin/alembic $IMAGE upgrade head +docker run --rm -t --net=host -e DRYDOCK_DB_URL="$DRYDOCK_DB_URL" -w /tmp/drydock --entrypoint /usr/local/bin/alembic $IMAGE upgrade head diff --git a/python/tests/postgres/stop_postgres.sh b/python/tests/postgres/stop_postgres.sh index bed5cd27..a14ddd70 100755 --- a/python/tests/postgres/stop_postgres.sh +++ b/python/tests/postgres/stop_postgres.sh @@ -1,2 +1,2 @@ #!/bin/bash -sudo docker stop 'psql_integration' +docker stop 'psql_integration' diff --git a/tools/drydock_image_run.sh b/tools/drydock_image_run.sh index c875499d..ddae3c34 100755 --- a/tools/drydock_image_run.sh +++ b/tools/drydock_image_run.sh @@ -2,14 +2,16 @@ set -x IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} +PSQL_CONTAINER_NAME=psql_integration_$(date +%Y%m%d%H%M%s%s) +DRYDOCK_CONTAINER_NAME=drydock_test_$(date +%Y%m%d%H%M%s%s) function start_db { - if [[ ! -z $(docker ps | grep 'psql_integration') ]] + if [[ ! -z $(docker ps | grep "${PSQL_CONTAINER_NAME}" ) ]] then - sudo docker stop 'psql_integration' + sudo docker stop "${PSQL_CONTAINER_NAME}" fi - docker run --rm -dp 5432:5432 --name 'psql_integration' -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6 + docker run --rm -dp 5432:5432 --name "${PSQL_CONTAINER_NAME}" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6 sleep 15 docker run --rm --net host postgres:14.6 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres @@ -47,7 +49,7 @@ function init_db { function test_drydock { TMPETC=$1 docker run \ - -d --name 'drydock_test' --net host \ + -d --name "${DRYDOCK_CONTAINER_NAME}" --net host \ -v ${TMPETC}:/etc/drydock \ ${IMAGE} @@ -55,6 +57,12 @@ function test_drydock { RESULT=$(curl --noproxy '*' -i 'http://127.0.0.1:9000/api/v1.0/tasks' | tr '\r' '\n' | head -1) GOOD="HTTP/1.1 200 OK" + if [[ "${RESULT}" != "${GOOD}" ]]; then + if docker exec -t ${CONTAINER_NAME} /bin/bash -c "curl -i 'http://127.0.0.1:9000/api/v1.0/tasks' --noproxy '*' | tr '\r' '\n' | head -1 "; then + RESULT="${GOOD}" + fi + fi + if [[ ${RESULT} == ${GOOD} ]] then RC=0 @@ -62,16 +70,16 @@ function test_drydock { RC=1 fi - docker logs drydock_test + docker logs "${DRYDOCK_CONTAINER_NAME}" return $RC } function cleanup { TMPDIR=$1 - docker stop psql_integration - docker stop drydock_test - docker rm drydock_test -# rm -rf $TMPDIR + docker stop "${PSQL_CONTAINER_NAME}" + docker stop "${DRYDOCK_CONTAINER_NAME}" + docker rm "${DRYDOCK_CONTAINER_NAME}" + rm -rf $TMPDIR } start_db diff --git a/tools/gate/playbooks/baclient_test.yaml b/tools/gate/playbooks/baclient_test.yaml new file mode 100644 index 00000000..a4331251 --- /dev/null +++ b/tools/gate/playbooks/baclient_test.yaml @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Combine several test tasks into a single playbook +# to minimize Zuul node consumption + +- hosts: primary + + + + roles: + - bindep + - ensure-docker + - ensure-python + - ensure-pip + - ensure-tox + + + tasks: + - name: Execute Go tests for baclient + make: + chdir: "{{ zuul.project.src_dir }}" + target: test_baclient + become: true + register: result + diff --git a/tools/gate/playbooks/build-charts.yaml b/tools/gate/playbooks/build-charts.yaml index d43ef828..6d4bddd9 100644 --- a/tools/gate/playbooks/build-charts.yaml +++ b/tools/gate/playbooks/build-charts.yaml @@ -23,6 +23,11 @@ - ensure-pip tasks: + - name: Execute the make target for Helm chart linting + make: + chdir: "{{ zuul.project.src_dir }}" + target: helm_lint + register: result - name: Execute the make target to package Helm charts. make: chdir: "{{ zuul.project.src_dir }}" diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml index 486a0e6e..49573316 100644 --- a/tools/gate/playbooks/docker-image-build.yaml +++ b/tools/gate/playbooks/docker-image-build.yaml @@ -56,6 +56,16 @@ executable: pip3 become: True + + - name: Run images + shell: | + make run_images + args: + chdir: "{{ zuul.project.src_dir }}" + executable: /bin/bash + become: True + + - name: Make images when: not publish block: @@ -69,12 +79,11 @@ - shell: "docker images" register: docker_images - - debug: var: docker_images - become: True + - name: Publish images block: - pip: diff --git a/tools/gate/playbooks/install-docker.yaml b/tools/gate/playbooks/install-docker.yaml new file mode 100644 index 00000000..8bb19111 --- /dev/null +++ b/tools/gate/playbooks/install-docker.yaml @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Combine several test tasks into a single playbook +# to minimize Zuul node consumption + +- hosts: primary + roles: + - bindep + - clear-firewall + - ensure-docker + tasks: + - name: Build images + shell: | + make images + args: + chdir: "{{ zuul.project.src_dir }}" + executable: /bin/bash + become: True \ No newline at end of file diff --git a/tools/gate/playbooks/omni_test.yaml b/tools/gate/playbooks/omni_test.yaml deleted file mode 100644 index 8c11ef80..00000000 --- a/tools/gate/playbooks/omni_test.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Combine several test tasks into a single playbook -# to minimize Zuul node consumption - -- hosts: primary - - - - roles: - - bindep - - ensure-docker - - ensure-python - - ensure-pip - - - tasks: - - name: Execute the make target for PEP8 linting - make: - chdir: "{{ zuul.project.src_dir }}" - target: pep8 - register: result - - name: Execute the make target for Helm chart linting - make: - chdir: "{{ zuul.project.src_dir }}" - target: helm_lint - register: result - - name: Execute the make target for security scanning - make: - chdir: "{{ zuul.project.src_dir }}" - target: security - register: result - - name: Execute the make target for unit testing - make: - chdir: "{{ zuul.project.src_dir }}" - target: unit_tests - register: result - - name: Execute Go tests for baclient - make: - chdir: "{{ zuul.project.src_dir }}" - target: test_baclient - become: true - register: result - - name: Setup Apparmor - shell: | - set -xe; - ./tools/deployment/apparmor/001-setup-apparmor-profiles.sh - args: - chdir: "{{ zuul.projects['opendev.org/openstack/openstack-helm-infra'].src_dir }}" - executable: /bin/bash - - name: Execute the make target for building and running the Drydock Docker image - make: - chdir: "{{ zuul.project.src_dir }}" - target: run_drydock - register: result - become: true \ No newline at end of file diff --git a/tox.ini b/tox.ini index 9521c0da..982e9895 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = py38,pep8,bandit setupdir=python/ - +minversion = 3.28.0 [testenv] setenv = YAMLDIR = {toxinidir}/python/tests/yaml_samples/ @@ -24,6 +24,21 @@ commands= rm -f python/requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen.txt" + +[testenv:freeze-bionic] +basepython=python3 +recreate = True +allowlist_externals= + rm + sh +deps= + -r{toxinidir}/python/requirements-direct.txt +commands= + rm -f python/requirements-frozen-bionic.txt + sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen-bionic.txt" + + + [testenv:yapf] allowlist_externals=find commands= @@ -43,6 +58,21 @@ commands= --cov=drydock_provisioner \ {toxinidir}/python/tests/unit/{posargs} +[testenv:py36] +usedevelop=True +setenv= + PYTHONWARNING=all + YAMLDIR={toxinidir}/python/tests/yaml_samples/ +deps= + -r{toxinidir}/python/requirements-frozen-bionic.txt + -r{toxinidir}/python/test-requirements.txt + tox +commands= + py.test \ + --cov=drydock_provisioner \ + {toxinidir}/python/tests/unit/{posargs} + + [testenv:integration] passenv=DOCKER_REGISTRY,IMAGE_NAME,IMAGE_PREFIX,IMAGE_TAG,DISTRO setenv= @@ -59,15 +89,22 @@ commands= [testenv:cover] usedevelop=True -passenv=DOCKER_REGISTRY,IMAGE_NAME,IMAGE_PREFIX,IMAGE_TAG +passenv=DOCKER_REGISTRY,IMAGE_NAME,IMAGE_PREFIX,IMAGE_TAG,DISTRO setenv= YAMLDIR={toxinidir}/python/tests/yaml_samples/ allowlist_externals= sh commands= sh -c {toxinidir}/python/tests/postgres/start_postgres.sh - py.test --cov=drydock_provisioner \ - {toxinidir}/python/tests/unit/ {toxinidir}/python/tests/integration/postgres + py.test \ + --cov=drydock_provisioner \ + --cov-report html:cover \ + --cov-report xml:cover/coverage.xml \ + --cov-report term \ + --cov-fail-under 50 \ + {toxinidir}/python/tests/unit/ \ + {toxinidir}/python/tests/integration/postgres + [testenv:genconfig] allowlist_externals = @@ -86,8 +123,9 @@ commands = sh -c 'oslopolicy-sample-generator --config-file etc/drydock/drydock-policy-generator.conf | tee etc/drydock/policy.yaml.sample doc/source/_static/policy.yaml.sample' [testenv:pep8] -commands = flake8 \ - {posargs} +commands = + flake8 {posargs} + bandit -r drydock_provisioner -n 5 [testenv:bandit] commands = bandit -r drydock_provisioner -n 5