diff --git a/.zuul.yaml b/.zuul.yaml index b330506c..d53f762b 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -13,6 +13,8 @@ - project: templates: - docs-on-readthedocs + - openstack-python38-jobs + - openstack-python36-jobs vars: rtd_webhook_id: '38809' rtd_project_name: 'airship-drydock' @@ -22,14 +24,18 @@ - airship-drydock-chart-build-gate - airship-drydock-chart-build-latest-htk - airship-drydock-docker-build-gate-ubuntu_focal + - airship-drydock-docker-build-gate-ubuntu_bionic + gate: jobs: - airship-drydock-omni-test - airship-drydock-chart-build-gate - airship-drydock-docker-build-gate-ubuntu_focal + - airship-drydock-docker-build-gate-ubuntu_bionic post: jobs: - airship-drydock-docker-publish-ubuntu_focal + - airship-drydock-docker-publish-ubuntu_bionic - drydock-upload-git-mirror - nodeset: @@ -90,6 +96,22 @@ patch_set: true +- job: + name: airship-drydock-docker-build-gate-ubuntu_bionic + timeout: 1800 + run: tools/gate/playbooks/docker-image-build.yaml + nodeset: airship-drydock-single-node + irrelevant-files: + - '^doc/.*' + - '^charts/.*' + vars: + publish: false + distro: ubuntu_bionic + tags: + dynamic: + patch_set: true + + - job: name: airship-drydock-docker-publish-ubuntu_focal timeout: 1800 @@ -109,6 +131,28 @@ commit: true static: - latest + +- job: + name: airship-drydock-docker-publish-ubuntu_bionic + timeout: 1800 + run: tools/gate/playbooks/docker-image-build.yaml + nodeset: airship-drydock-single-node-focal + secrets: + - airship_drydock_quay_creds + irrelevant-files: + - '^doc/.*' + - '^charts/.*' + vars: + publish: true + distro: ubuntu_bionic + tags: + dynamic: + branch: true + commit: true + static: + - latest + + - secret: name: airship_drydock_quay_creds data: @@ -143,15 +187,15 @@ git_mirror_repository: airshipit/drydock secrets: - name: git_mirror_credentials - secret: drydock-airshipit-github-secret + secret: drydock-airshipit-github-secret-2023-03-23 pass-to-parent: true - secret: - name: drydock-airshipit-github-secret + name: drydock-airshipit-github-secret-2023-03-23 data: user: git host: github.com - host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== + host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= ssh_key: !encrypted/pkcs1-oaep - UjLtbrpoJqWLQQBAjOWgiHPpJP6WqYenCJYtkv10ayR0esOS0BOUVSOCUC2TC+fBKx/rn dREF8XuQSDH2gmSUvTafzMhGZmy+Rk/KOi2Et+V3uNCdczxFPEovliQCQmt2LwZADubMg diff --git a/Makefile b/Makefile index ace042ed..82d3d80c 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ BUILD_DIR := $(shell mkdir -p build && mktemp -d -p build) DOCKER_REGISTRY ?= quay.io IMAGE_NAME ?= drydock IMAGE_PREFIX ?= airshipit -IMAGE_TAG ?= dev +IMAGE_TAG ?= latest HELM := $(shell realpath $(BUILD_DIR))/helm UBUNTU_BASE_IMAGE ?= DISTRO ?= ubuntu_focal @@ -35,7 +35,7 @@ export images: build_drydock # Run an image locally and exercise simple tests -run_images: run_drydock +run_images: build_drydock run_drydock # Run tests tests: pep8 security docs unit_tests test_baclient @@ -57,7 +57,7 @@ req_freeze: tox -re freeze # Run the drydock container and exercise simple tests -run_drydock: build_drydock +run_drydock: tools/drydock_image_run.sh # It seems CICD expects the target 'drydock' to diff --git a/images/drydock/Dockerfile.ubuntu_bionic b/images/drydock/Dockerfile.ubuntu_bionic index b17c9340..8943ecd6 100644 --- a/images/drydock/Dockerfile.ubuntu_bionic +++ b/images/drydock/Dockerfile.ubuntu_bionic @@ -23,13 +23,18 @@ ENV container docker ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 -# Copy direct dependency requirements only to build a dependency layer -RUN echo "deb ${UBUNTU_REPO} bionic main restricted universe multiverse" > /etc/apt/sources.list; \ - echo "deb ${UBUNTU_REPO} bionic-security main restricted universe multiverse" >> /etc/apt/sources.list; \ - echo "deb ${UBUNTU_REPO} bionic-updates main restricted universe multiverse" >> /etc/apt/sources.list; \ - cat /etc/apt/sources.list; \ - echo "APT::Get::AllowUnauthenticated ${ALLOW_UNAUTHENTICATED};" >> /etc/apt/apt.conf.d/00-local-mirrors; - +COPY ./bindep.txt /tmp/drydock/ +WORKDIR /tmp/drydock +RUN apt update \ + && apt install -y --allow-downgrades \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + python3-setuptools \ + && DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y tzdata \ + && pip3 install bindep \ + && bindep -f /tmp/drydock/bindep.txt --brief | xargs apt install -y COPY ./tools/baclient_build.sh /tmp/drydock/ COPY ./go /tmp/drydock/go @@ -73,29 +78,36 @@ ENV PORT 9000 ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 -# Copy direct dependency requirements only to build a dependency layer -RUN echo "deb ${UBUNTU_REPO} bionic main restricted universe multiverse" > /etc/apt/sources.list; \ - echo "deb ${UBUNTU_REPO} bionic-security main restricted universe multiverse" >> /etc/apt/sources.list; \ - echo "deb ${UBUNTU_REPO} bionic-updates main restricted universe multiverse" >> /etc/apt/sources.list; \ - cat /etc/apt/sources.list; \ - echo "APT::Get::AllowUnauthenticated ${ALLOW_UNAUTHENTICATED};" >> /etc/apt/apt.conf.d/00-local-mirrors; - - -COPY ./requirements-host.txt /tmp/drydock/ -COPY ./hostdeps.sh /tmp/drydock -WORKDIR /tmp/drydock -RUN ./hostdeps.sh; \ - rm -r /var/lib/apt/lists/* +RUN DEBIAN_FRONTEND=noninteractive \ + apt update \ + && DEBIAN_FRONTEND=noninteractive \ + apt install -y \ + --allow-downgrades \ + --no-install-recommends \ + python3-dev \ + python3-pip \ + python3-setuptools \ + pkg-config \ + libvirt-dev \ + libssl-dev \ + libpq-dev \ + gcc \ + ssh\ + curl \ + netbase \ + && rm -r /var/lib/apt/lists/* # Install LibYAML ENV LD_LIBRARY_PATH=/usr/local/lib COPY --from=baclient_builder /usr/local/lib /usr/local/lib COPY --from=baclient_builder /usr/local/include/yaml.h /usr/local/include/yaml.h -COPY ./python/requirements-frozen.txt /tmp/drydock/ -RUN pip3 install \ +COPY ./python/requirements-frozen-bionic.txt /tmp/drydock/ +RUN \ + pip3 install --upgrade pip \ + && pip3 install \ --no-cache-dir \ - -r /tmp/drydock/requirements-frozen.txt + -r /tmp/drydock/requirements-frozen-bionic.txt COPY ./python /tmp/drydock/python WORKDIR /tmp/drydock/python diff --git a/images/drydock/Dockerfile.ubuntu_focal b/images/drydock/Dockerfile.ubuntu_focal index 933a2a75..5ed834dc 100644 --- a/images/drydock/Dockerfile.ubuntu_focal +++ b/images/drydock/Dockerfile.ubuntu_focal @@ -95,6 +95,7 @@ RUN DEBIAN_FRONTEND=noninteractive \ pkg-config \ libvirt-dev \ libssl-dev \ + libpq-dev \ gcc \ ssh\ curl \ @@ -107,7 +108,9 @@ COPY --from=baclient_builder /usr/local/lib /usr/local/lib COPY --from=baclient_builder /usr/local/include/yaml.h /usr/local/include/yaml.h COPY ./python/requirements-frozen.txt /tmp/drydock/ -RUN pip3 install \ +RUN \ + pip3 install --upgrade pip \ + && pip3 install \ --no-cache-dir \ -r /tmp/drydock/requirements-frozen.txt diff --git a/python/requirements-frozen-bionic.txt b/python/requirements-frozen-bionic.txt new file mode 100644 index 00000000..382b98cc --- /dev/null +++ b/python/requirements-frozen-bionic.txt @@ -0,0 +1,99 @@ +alembic==1.7.7 +amqp==5.1.1 +attrs==22.2.0 +bcrypt==4.0.1 +Beaker==1.12.1 +cached-property==1.5.2 +cachetools==4.2.4 +certifi==2023.5.7 +cffi==1.15.1 +charset-normalizer==2.0.12 +click==8.0.4 +cryptography==40.0.2 +debtcollector==2.5.0 +decorator==5.1.1 +defusedxml==0.7.1 +dnspython==2.2.1 +dogpile.cache==1.2.1 +eventlet==0.33.3 +falcon==3.1.1 +fasteners==0.18 +fixtures==4.0.1 +futurist==2.4.1 +greenlet==2.0.2 +idna==3.4 +importlib-metadata==4.8.3 +importlib-resources==5.4.0 +iso8601==1.1.0 +Jinja2==3.0.3 +jsonpatch==1.32 +jsonpath-rw==1.4.0 +jsonpointer==2.3 +jsonschema==3.2.0 +keystoneauth1==5.1.2 +keystonemiddleware==9.5.0 +kombu==5.1.0 +libvirt-python==9.3.0 +Mako==1.1.6 +MarkupSafe==2.0.1 +msgpack==1.0.5 +netaddr==0.8.0 +netifaces==0.11.0 +oauthlib==3.2.2 +os-service-types==1.7.0 +oslo.cache==2.11.0 +oslo.concurrency==4.5.1 +oslo.config==8.8.0 +oslo.context==4.1.0 +oslo.i18n==5.1.0 +oslo.log==4.8.0 +oslo.messaging==12.14.0 +oslo.metrics==0.4.0 +oslo.middleware==4.5.1 +oslo.policy==3.12.1 +oslo.serialization==4.3.0 +oslo.service==2.8.0 +oslo.utils==4.13.0 +oslo.versionedobjects==2.6.0 +packaging==21.3 +Paste==3.5.3 +PasteDeploy==2.1.1 +pbr==5.11.1 +pip==21.3.1 +ply==3.11 +prometheus-client==0.16.0 +psycopg2-binary==2.9.6 +PTable==0.9.2 +pycadf==3.1.1 +pycparser==2.21 +pyghmi==1.5.61 +pylibyaml==0.1.0 +pymongo==4.1.1 +pyparsing==3.0.9 +pyrsistent==0.18.0 +python-dateutil==2.8.2 +python-keystoneclient==4.5.0 +pytz==2023.3 +PyYAML==6.0 +redfish==3.1.9 +repoze.lru==0.7 +requests==2.27.1 +requests-toolbelt==1.0.0 +requests-unixsocket==0.3.0 +rfc3986==1.5.0 +Routes==2.5.1 +setuptools==59.6.0 +six==1.16.0 +SQLAlchemy==1.3.20 +statsd==4.0.1 +stevedore==3.5.2 +typing_extensions==4.1.1 +ulid2==0.3.0 +urllib3==1.26.16 +uWSGI==2.0.21 +vine==5.0.0 +WebOb==1.8.7 +wheel==0.37.1 +wrapt==1.15.0 +yappi==1.4.0 +zipp==3.6.0 diff --git a/python/test-requirements.txt b/python/test-requirements.txt index 0ea77b9e..92e9d790 100644 --- a/python/test-requirements.txt +++ b/python/test-requirements.txt @@ -1,19 +1,18 @@ # tests -click==6.7 +click falcon==3.1.1 jsonschema==3.2.0 mock==5.0.1 pylibyaml==0.1.0 -pymongo==3.10.1 +pymongo pytest >= 3.0 pytest-cov==4.0.0 pytest-mock -responses==0.23.1 -setuptools==56.0.0 -ulid2==0.1.1 +responses +ulid2 # tools yapf flake8 -bandit>=1.1.0 -sphinx>=1.6.2 +bandit==1.6.0 +sphinx<7.0.0 diff --git a/tools/drydock_image_run.sh b/tools/drydock_image_run.sh index c875499d..ddae3c34 100755 --- a/tools/drydock_image_run.sh +++ b/tools/drydock_image_run.sh @@ -2,14 +2,16 @@ set -x IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO} +PSQL_CONTAINER_NAME=psql_integration_$(date +%Y%m%d%H%M%s%s) +DRYDOCK_CONTAINER_NAME=drydock_test_$(date +%Y%m%d%H%M%s%s) function start_db { - if [[ ! -z $(docker ps | grep 'psql_integration') ]] + if [[ ! -z $(docker ps | grep "${PSQL_CONTAINER_NAME}" ) ]] then - sudo docker stop 'psql_integration' + sudo docker stop "${PSQL_CONTAINER_NAME}" fi - docker run --rm -dp 5432:5432 --name 'psql_integration' -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6 + docker run --rm -dp 5432:5432 --name "${PSQL_CONTAINER_NAME}" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:14.6 sleep 15 docker run --rm --net host postgres:14.6 psql -h localhost -c "create user drydock with password 'drydock';" postgres postgres @@ -47,7 +49,7 @@ function init_db { function test_drydock { TMPETC=$1 docker run \ - -d --name 'drydock_test' --net host \ + -d --name "${DRYDOCK_CONTAINER_NAME}" --net host \ -v ${TMPETC}:/etc/drydock \ ${IMAGE} @@ -55,6 +57,12 @@ function test_drydock { RESULT=$(curl --noproxy '*' -i 'http://127.0.0.1:9000/api/v1.0/tasks' | tr '\r' '\n' | head -1) GOOD="HTTP/1.1 200 OK" + if [[ "${RESULT}" != "${GOOD}" ]]; then + if docker exec -t ${CONTAINER_NAME} /bin/bash -c "curl -i 'http://127.0.0.1:9000/api/v1.0/tasks' --noproxy '*' | tr '\r' '\n' | head -1 "; then + RESULT="${GOOD}" + fi + fi + if [[ ${RESULT} == ${GOOD} ]] then RC=0 @@ -62,16 +70,16 @@ function test_drydock { RC=1 fi - docker logs drydock_test + docker logs "${DRYDOCK_CONTAINER_NAME}" return $RC } function cleanup { TMPDIR=$1 - docker stop psql_integration - docker stop drydock_test - docker rm drydock_test -# rm -rf $TMPDIR + docker stop "${PSQL_CONTAINER_NAME}" + docker stop "${DRYDOCK_CONTAINER_NAME}" + docker rm "${DRYDOCK_CONTAINER_NAME}" + rm -rf $TMPDIR } start_db diff --git a/tools/gate/playbooks/docker-image-build.yaml b/tools/gate/playbooks/docker-image-build.yaml index 486a0e6e..be5387c4 100644 --- a/tools/gate/playbooks/docker-image-build.yaml +++ b/tools/gate/playbooks/docker-image-build.yaml @@ -69,12 +69,20 @@ - shell: "docker images" register: docker_images - - debug: var: docker_images become: True + - name: Run images + shell: | + make run_drydock + args: + chdir: "{{ zuul.project.src_dir }}" + executable: /bin/bash + become: True + + - name: Publish images block: - pip: diff --git a/tox.ini b/tox.ini index 9521c0da..00684d6c 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,21 @@ commands= rm -f python/requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen.txt" + +[testenv:freeze-bionic] +basepython=python3 +recreate = True +allowlist_externals= + rm + sh +deps= + -r{toxinidir}/python/requirements-direct.txt +commands= + rm -f python/requirements-frozen-bionic.txt + sh -c "pip freeze --all | grep -vE 'drydock_provisioner|pyinotify|pkg-resources==0.0.0' > python/requirements-frozen-bionic.txt" + + + [testenv:yapf] allowlist_externals=find commands= @@ -43,6 +58,20 @@ commands= --cov=drydock_provisioner \ {toxinidir}/python/tests/unit/{posargs} +[testenv:py36] +usedevelop=True +setenv= + PYTHONWARNING=all + YAMLDIR={toxinidir}/python/tests/yaml_samples/ +deps= + -r{toxinidir}/python/requirements-frozen-bionic.txt + -r{toxinidir}/python/test-requirements.txt +commands= + py.test \ + --cov=drydock_provisioner \ + {toxinidir}/python/tests/unit/{posargs} + + [testenv:integration] passenv=DOCKER_REGISTRY,IMAGE_NAME,IMAGE_PREFIX,IMAGE_TAG,DISTRO setenv=