Restored ubuntu_bionic image build

This PS restores image build for ubuntu_bionic and adds appropriate
gates to keep it tested by appropriate functional and integrational
tests.

Change-Id: I4c4c7186e67c771f8249b2017b2507aac17760b7
This commit is contained in:
Sergiy Markin 2023-05-26 15:52:13 +00:00
parent 9c28c832dd
commit 7ea83c25ef
15 changed files with 608 additions and 62 deletions

View File

@ -14,6 +14,7 @@
templates:
- docs-on-readthedocs
- openstack-python38-jobs
- openstack-python36-jobs
vars:
rtd_webhook_id: '38576'
rtd_project_name: 'airship-shipyard'
@ -21,20 +22,28 @@
check:
jobs:
- openstack-tox-pep8
- airship-shipyard-openstack-tox-cover-focal
- airship-shipyard-chart-build-gate
- airship-shipyard-chart-build-latest-htk
- airship-shipyard-whitespace-lint-gate
- airship-shipyard-airskiff-deployment
- airship-shipyard-airskiff-deployment-bionic
- airship-shipyard-airskiff-deployment-focal
- airship-shipyard-image-gate-ubuntu_focal
- airship-shipyard-image-gate-ubuntu_bionic
gate:
jobs:
- openstack-tox-pep8
- airship-shipyard-openstack-tox-cover-focal
- airship-shipyard-chart-build-gate
- airship-shipyard-whitespace-lint-gate
- airship-shipyard-airskiff-deployment-bionic
- airship-shipyard-airskiff-deployment-focal
- airship-shipyard-image-gate-ubuntu_focal
- airship-shipyard-image-gate-ubuntu_bionic
post:
jobs:
- airship-shipyard-docker-build-post-ubuntu_focal
- airship-shipyard-docker-build-publish-ubuntu_focal
- airship-shipyard-docker-build-publish-ubuntu_bionic
- shipyard-upload-git-mirror
- nodeset:
@ -49,12 +58,21 @@
- name: primary
label: ubuntu-focal
- job:
name: airship-shipyard-openstack-tox-cover-focal
parent: openstack-tox-cover
description: Runs cover job on focal
nodeset: airship-shipyard-single-node-focal
- job:
name: airship-shipyard-chart-build-gate
description: Build charts using pinned Helm toolkit.
timeout: 900
run: tools/gate/playbooks/build-charts.yaml
nodeset: airship-shipyard-single-node-focal
vars:
HTK_COMMIT: a7cd689280cdbc0acd04a7a1b745941260e8700b
- job:
name: airship-shipyard-chart-build-latest-htk
@ -78,13 +96,14 @@
description: |
Base job for running airship-shipyard image related jobs.
timeout: 3600
run: tools/gate/playbooks/run-image.yaml
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: airship-shipyard-single-node-focal
irrelevant-files:
irrelevant-files: &irrelevant-files
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
- job:
name: airship-shipyard-image-gate-ubuntu_focal
description: |
@ -98,7 +117,20 @@
patch_set: true
- job:
name: airship-shipyard-airskiff-deployment
name: airship-shipyard-image-gate-ubuntu_bionic
description: |
Run shipyard-image build for ubuntu_bionic
parent: airship-shipyard-image-base
irrelevant-files: *irrelevant-files
vars:
publish: false
distro: ubuntu_bionic
tags:
dynamic:
patch_set: true
- job:
name: airship-shipyard-airskiff-deployment-focal
nodeset: airship-shipyard-single-node-focal
description: |
Deploy Memcached using Airskiff and submitted Armada changes.
@ -114,29 +146,35 @@
vars:
CLONE_SHIPYARD: false
HTK_COMMIT: master
irrelevant-files: &irrelevant-files
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
SHIPYARD_IMAGE_DISTRO: ubuntu_focal
irrelevant-files: *irrelevant-files
- job:
name: airship-shipyard-image-gate-opensuse
name: airship-shipyard-airskiff-deployment-bionic
nodeset: airship-shipyard-single-node-focal
description: |
Run shipyard-image build for opensuse.
parent: airship-shipyard-image-base
Deploy Memcached using Airskiff and submitted Armada changes.
timeout: 9600
voting: true
pre-run:
- tools/gate/playbooks/airskiff-reduce-site.yaml
- tools/gate/playbooks/git-config.yaml
run: tools/gate/playbooks/airskiff-deploy.yaml
post-run: tools/gate/playbooks/debug-report.yaml
required-projects:
- airship/treasuremap
vars:
publish: false
distro: opensuse_15
tags:
dynamic:
patch_set: true
CLONE_SHIPYARD: false
HTK_COMMIT: master
SHIPYARD_IMAGE_DISTRO: ubuntu_bionic
irrelevant-files: *irrelevant-files
- job:
name: airship-shipyard-docker-build-post-ubuntu_focal
name: airship-shipyard-docker-build-publish-ubuntu_focal
timeout: 1800
run: tools/gate/playbooks/run-image.yaml
nodeset: airship-shipyard-single-node
nodeset: airship-shipyard-single-node-focal
irrelevant-files: *irrelevant-files
secrets:
- airship_shipyard_quay_creds
@ -151,19 +189,16 @@
- latest
- job:
name: airship-shipyard-docker-build-post-opensuse
name: airship-shipyard-docker-build-publish-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/run-image.yaml
nodeset: airship-shipyard-single-node
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
nodeset: airship-shipyard-single-node-focal
irrelevant-files: *irrelevant-files
secrets:
- airship_shipyard_quay_creds
vars:
publish: true
distro: opensuse_15
distro: ubuntu_bionic
tags:
dynamic:
branch: true
@ -171,6 +206,7 @@
static:
- latest
- secret:
name: airship_shipyard_quay_creds
data:
@ -205,15 +241,15 @@
git_mirror_repository: airshipit/shipyard
secrets:
- name: git_mirror_credentials
secret: shipyard-airshipit-github-secret
secret: shipyard-airshipit-github-secret-2023-03-23
pass-to-parent: true
- secret:
name: shipyard-airshipit-github-secret
name: shipyard-airshipit-github-secret-2023-03-23
data:
user: git
host: github.com
host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
host_key: github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
ssh_key: !encrypted/pkcs1-oaep
- hUsE3gOWqb6aoNVSpqRHcUbqi0wOjRJ47viZWr7JopICoyhIerZ6DN4dGRaFjJ+8UtQvD
uXKYHZ35+YbsJDQttIadtVvj5Va8hIToI/rARSktyDp4zb+fK829INKuOjSszowhGoDuv

View File

@ -40,18 +40,36 @@ IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO}
IMAGE_DIR:=images/$(IMAGE_NAME)
.PHONY: images
#Build all images in the list
images: $(IMAGE_NAME)
#Build and run all images in list
#sudo make images IMAGE_NAME=airflow will Build and Run airflow
#sudo make images will build and run airflow and shipyard
$(IMAGE_NAME):
@echo
@echo "===== Processing [$@] image ====="
@make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/$@ IMAGE_NAME=$@
@make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/$@_image_run.sh
# Build all docker images for this project
images: build_images
build_images: build_airflow build_shipyard
run_images: build_airflow run_airflow build_shipyard run_shipyard
#Build all images in list
build_airflow:
@echo
@echo "===== Processing [airflow] image ====="
@make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/airflow:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/airflow IMAGE_NAME=airflow
build_shipyard:
@echo
@echo "===== Processing [shipyard] image ====="
@make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/shipyard:${IMAGE_TAG}-${DISTRO} IMAGE_DIR=images/shipyard IMAGE_NAME=shipyard
#Run all images in list
run_airflow:
@echo
@echo "===== Processing [airflow] image ====="
@make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/airflow:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/airflow_image_run.sh
run_shipyard:
@echo
@echo "===== Processing [shipyard] image ====="
@make run IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/shipyard:${IMAGE_TAG}-${DISTRO} SCRIPT=./tools/shipyard_image_run.sh
# Create tgz of the chart
.PHONY: charts

View File

@ -64,10 +64,13 @@ RUN set -ex && \
libffi-dev \
libssl-dev \
libpq-dev \
libpq5 \
libtool \
locales \
netcat \
netbase \
postgresql-client \
postgresql-common \
python3 \
python3-setuptools \
python3-pip \
@ -110,7 +113,7 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
# Note - removing snakebite (python 2 vs. 3). See:
# https://github.com/puckel/docker-airflow/issues/77
# Install Airflow directly to allow overriding source
COPY images/airflow/requirements.txt /tmp/
COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /tmp/requirements.txt
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
&& pip3 install $AIRFLOW_SRC --no-cache-dir \
&& (pip3 uninstall -y snakebite || true) \
@ -135,8 +138,8 @@ ENV PBR_VERSION 0.1a1
# Since Shipyard and Airflow are built together as images, this should prevent
# stale or out-of-date code between these parts.
# Shipyard requirements, source and installation
COPY ${ctx_base}/shipyard_airflow/requirements.txt /tmp/api_requirements.txt
RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
# COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /tmp/api_requirements.txt
# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
RUN cd /tmp/shipyard \

View File

@ -148,8 +148,8 @@ ENV PBR_VERSION 0.1a1
# Since Shipyard and Airflow are built together as images, this should prevent
# stale or out-of-date code between these parts.
# Shipyard requirements, source and installation
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt
RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
# COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt
# RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
RUN cd /tmp/shipyard \

View File

@ -85,8 +85,8 @@ RUN chown -R shipyard: /home/shipyard \
&& chmod +x /home/shipyard/entrypoint.sh
# Requirements and Shipyard source
COPY ${ctx_base}/shipyard_airflow/requirements.txt /home/shipyard/api_requirements.txt
COPY ${ctx_base}/shipyard_client/requirements.txt /home/shipyard/client_requirements.txt
COPY ${ctx_base}/shipyard_airflow/requirements-frozen-bionic.txt /home/shipyard/api_requirements.txt
COPY ${ctx_base}/shipyard_client/requirements-frozen-bionic.txt /home/shipyard/client_requirements.txt
COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/
COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/

View File

@ -34,7 +34,7 @@ WTForms<=2.3.3
itsdangerous==2.0.1
git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e#egg=deckhand
git+https://opendev.org/airship/drydock.git@375abedb8aa3a413ca70a47aef467f39d65e1aee#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@4be7fa44a743fbcdf792e2bdca3af6c95b540f97#egg=armada
git+https://opendev.org/airship/promenade.git@45bcba068eb42fe6ba54d494b12122600dcb2c6c#egg=promenade
git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5#egg=deckhand
git+https://opendev.org/airship/drydock.git@73d22a488f7f5b7217e86e86da9f38b5b9085009#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@812546c87595b1d8a1bc0da931d644473b03b31f#egg=armada
git+https://opendev.org/airship/promenade.git@2f2455f0d159287565a279a99c0bbeb235e81ffb#egg=promenade

View File

@ -0,0 +1,241 @@
alabaster==0.7.13
alembic==1.0.1
amqp==2.6.1
apache-airflow==1.10.5
apispec==1.3.3
argcomplete==3.0.8
Armada @ git+https://opendev.org/airship/armada.git@812546c87595b1d8a1bc0da931d644473b03b31f
arrow==0.17.0
attrs==22.2.0
autopage==0.5.1
Babel==2.11.0
bcrypt==4.0.1
Beaker==1.12.0
billiard==3.6.4.0
cached-property==1.5.2
cachetools==4.2.4
celery==4.3.0
certifi==2023.5.7
cffi==1.15.1
chardet==3.0.4
charset-normalizer==3.0.1
click==7.1.2
cliff==3.10.1
cmd2==2.4.3
colorama==0.4.5
colorlog==4.0.2
configparser==3.5.3
coverage==5.3
croniter==0.3.37
cryptography==3.4.8
debtcollector==2.5.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5
decorator==4.4.2
deepdiff==5.7.0
defusedxml==0.7.1
dill==0.2.9
distlib==0.3.6
dnspython==2.2.1
docutils==0.17
dogpile.cache==1.2.1
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@73d22a488f7f5b7217e86e86da9f38b5b9085009#subdirectory=python
dulwich==0.20.50
dumb-init==1.2.5.post1
eventlet==0.33.3
extras==1.0.0
falcon==3.1.1
fasteners==0.18
filelock==3.4.1
fixtures==3.0.0
flake8==3.8.4
Flask==1.1.0
Flask-Admin==1.5.3
Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@b71789d85b632935eca79c0b53fb77f20bf17fe6
Flask-Babel==0.12.2
Flask-Caching==1.3.3
Flask-JWT-Extended==3.25.1
Flask-Login==0.4.1
Flask-OpenID==1.3.0
Flask-SQLAlchemy==2.5.1
flask-swagger==0.2.13
Flask-WTF==0.14.3
flower==0.9.7
funcsigs==1.0.0
future==0.16.0
futurist==2.4.1
gitdb==4.0.9
GitPython==3.1.18
google-auth==2.19.0
greenlet==2.0.2
gunicorn==19.10.0
hacking==4.1.0
hmsclient==0.1.1
html5lib==0.9999999
httpexceptor==1.4.0
humanize==3.14.0
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
importlib-resources==5.4.0
iso8601==1.1.0
itsdangerous==2.0.1
JayDeBeApi==1.2.3
Jinja2==2.10.3
JPype1==1.3.0
json-merge-patch==0.2
jsonpath-ng==1.5.3
jsonpath-rw==1.4.0
jsonpath-rw-ext==1.2.2
jsonpickle==1.4.1
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==9.5.0
kombu==4.6.3
kubernetes==26.1.0
lazy-object-proxy==1.7.1
lockfile==0.12.2
Mako==1.1.6
Markdown==2.6.11
MarkupSafe==2.0.1
marshmallow==2.19.5
marshmallow-enum==1.5.1
marshmallow-sqlalchemy==0.18.0
mccabe==0.6.1
mock==5.0.2
msgpack==1.0.5
natsort==8.2.0
netaddr==0.8.0
netifaces==0.11.0
networkx==2.5.1
nose==1.3.7
numpy==1.19.5
oauthlib==3.2.2
ordered-set==4.0.2
ordereddict==1.1
os-service-types==1.7.0
oslo.cache==2.10.1
oslo.concurrency==4.5.1
oslo.config==8.7.1
oslo.context==4.1.0
oslo.db==10.0.0
oslo.i18n==5.1.0
oslo.log==4.6.0
oslo.messaging==10.3.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==4.2.0
oslo.service==2.8.0
oslo.utils==4.12.3
packaging==21.3
pandas==0.25.3
Paste==3.5.0
PasteDeploy==2.1.1
PasteScript==3.3.0
pbr==5.5.1
pendulum==1.4.4
pip==21.3.1
platformdirs==2.4.0
ply==3.11
prettytable==2.5.0
prison==0.1.0
promenade @ git+https://opendev.org/airship/promenade.git@2f2455f0d159287565a279a99c0bbeb235e81ffb
prometheus-client==0.8.0
protobuf==3.19.6
psutil==5.9.5
psycopg2-binary==2.8.4
py==1.11.0
pyarrow==6.0.1
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycadf==3.1.1
pycodestyle==2.6.0
pycparser==2.21
pyflakes==2.2.0
Pygments==2.14.0
PyHive==0.6.5
PyJWT==1.7.1
pylibyaml==0.1.0
pymongo==4.1.1
pyparsing==2.4.7
pyperclip==1.8.2
pyproject-api==0.1.0
pyrsistent==0.18.0
python-barbicanclient==5.2.0
python-daemon==2.1.2
python-dateutil==2.8.1
python-editor==1.0.4
python-keystoneclient==4.5.0
python-memcached==1.59
python-mimeparse==1.6.0
python-subunit==1.4.0
python3-openid==3.2.0
pytz==2018.5
pytzdata==2020.1
PyYAML==5.4.1
regex==2020.11.13
reno==4.0.0
repoze.lru==0.7
requests==2.23.0
requests-oauthlib==1.3.1
resolver==0.2.1
responses==0.12.1
retry==0.9.2
rfc3986==1.5.0
Routes==2.5.1
rsa==4.9
selector==0.10.1
setproctitle==1.2.3
setuptools==45.2.0
simplejson==3.19.1
six==1.16.0
smmap==5.0.0
snakebite==2.11.0
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
SQLAlchemy==1.3.15
sqlalchemy-migrate==0.13.0
sqlparse==0.4.4
statsd==4.0.1
stevedore==3.5.2
tabulate==0.8.3
Tempita==0.5.2
tenacity==4.12.0
termcolor==1.1.0
testrepository==0.0.20
testresources==2.0.1
testscenarios==0.5.0
testtools==2.5.0
text-unidecode==1.2
thrift==0.16.0
tiddlyweb==2.4.3
tomli==1.2.3
tomlkit==0.11.6
tornado==5.1.1
typing-extensions==3.7.4.3
tzlocal==1.5.1
ulid==1.1
unicodecsv==0.14.1
urllib3==1.25.11
uWSGI==2.0.21
vine==1.3.0
virtualenv==20.16.2
wcwidth==0.2.6
WebOb==1.8.7
websocket-client==1.3.1
Werkzeug==0.16.1
wheel==0.37.1
wrapt==1.15.0
wsgi-intercept==1.11.0
WTForms==2.3.3
xmltodict==0.13.0
yappi==1.4.0
yq==3.2.1
zipp==3.6.0
zope.deprecation==4.4.0

View File

@ -12,6 +12,7 @@ classifier =
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.10

View File

@ -12,6 +12,17 @@ deps =
-r{toxinidir}/test-requirements.txt
[testenv:freeze-bionic]
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
commands=
rm -f {toxinidir}/requirements-frozen-bionic.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen-bionic.txt"
[testenv:freeze]
recreate = True
allowlist_externals=
@ -28,11 +39,28 @@ commands=
skipsdist=True
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
basepython=python3.8
allowlist_externals=
bash
airflow
commands =
bash -c "rm -f ${HOME}/airflow/airflow.db"
bash -c "rm -f $HOME/airflow/airflow.db"
pip install -r{toxinidir}/test-requirements.txt
airflow initdb
airflow run example_bash_operator runme_0 2018-01-01
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow dag_state example_bash_operator 2018-01-01
pytest {posargs} -vv
[testenv:cover]
skipsdist=True
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
allowlist_externals=
bash
airflow
commands =
bash -c "rm -f $HOME/airflow/airflow.db"
pip install -r{toxinidir}/test-requirements.txt
airflow initdb
airflow run example_bash_operator runme_0 2018-01-01
@ -44,9 +72,31 @@ commands =
--cov-report=term-missing:skip-covered \
--cov-config=.coveragerc \
--cov=shipyard_airflow \
--cov-report=html \
--cov-report html:cover \
--cov-report xml:cover/coverage.xml \
--cov-report term \
-vv
[testenv:py36]
skipsdist=True
basepython=python3.6
deps =
-r{toxinidir}/requirements-frozen-bionic.txt
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
allowlist_externals=
bash
airflow
commands =
bash -c "rm -f $HOME/airflow/airflow.db"
pip install -r{toxinidir}/test-requirements.txt
airflow initdb
airflow run example_bash_operator runme_0 2018-01-01
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow dag_state example_bash_operator 2018-01-01
pytest {posargs} -vv
[testenv:pep8]
deps=
flake8>=3.3.0

View File

@ -0,0 +1,30 @@
arrow==0.17.0
certifi==2023.5.7
chardet==3.0.4
click==7.1.2
click-default-group==1.2
coverage==5.3
Flask==1.1.0
idna==2.10
importlib-metadata==4.8.3
iso8601==1.1.0
itsdangerous==2.0.1
Jinja2==2.10.3
keystoneauth1==5.1.1
MarkupSafe==2.0.1
os-service-types==1.7.0
pbr==5.5.1
pip==21.3.1
PTable==0.9.2
pylibyaml==0.1.0
python-dateutil==2.8.1
PyYAML==5.4.1
requests==2.23.0
setuptools==45.2.0
six==1.16.0
stevedore==3.5.2
typing_extensions==4.1.1
urllib3==1.25.11
Werkzeug==0.16.1
wheel==0.37.1
zipp==3.6.0

View File

@ -7,8 +7,21 @@ setenv=
PYTHONWARNING=all
LC_ALL=C.UTF-8
LC_ALL=C.UTF-8
deps = -r{toxinidir}/requirements-frozen.txt
-r{toxinidir}/test-requirements.txt
deps =
-r{toxinidir}/requirements-frozen.txt
-r{toxinidir}/test-requirements.txt
[testenv:freeze-bionic]
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
commands=
rm -f {toxinidir}/requirements-frozen-bionic.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen-bionic.txt"
[testenv:freeze]
recreate = True
@ -21,9 +34,25 @@ commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
[testenv:py36]
skipsdist=True
deps =
-r{toxinidir}/requirements-frozen-bionic.txt
-r{toxinidir}/test-requirements.txt
commands =
pytest \
{posargs} \
-vv
[testenv:py38]
skipsdist=True
commands =
pytest \
{posargs} \
-vv
[testenv:cover]
skipsdist=True
commands =
pytest \
{posargs} \
@ -31,9 +60,12 @@ commands =
--cov-report=term-missing:skip-covered \
--cov-config=.coveragerc \
--cov=shipyard_client \
--cov-report=html \
--cov-report html:cover \
--cov-report xml:cover/coverage.xml \
--cov-report term \
-vv
[testenv:pep8]
deps=
flake8>=3.3.0

View File

@ -45,13 +45,15 @@
- name: Build Shipyard and Airflow with submitted changes
shell: |
make
docker pull quay.io/airshipit/shipyard:latest-ubuntu_bionic
docker rm quay.io/airshipit/shipyard:latest-ubuntu_bionic
docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal quay.io/airshipit/shipyard:latest-ubuntu_bionic
docker pull quay.io/airshipit/airflow:latest-ubuntu_bionic
docker rm quay.io/airshipit/airflow:latest-ubuntu_bionic
docker tag quay.io/airshipit/airflow:latest-ubuntu_focal quay.io/airshipit/airflow:latest-ubuntu_bionic
set -ex
export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
make images
if test "${DISTRO}" = 'ubuntu_bionic'
then
# this trick is needed to use bionic image instead of focal in airskiff deployment test
docker tag quay.io/airshipit/airflow:latest-ubuntu_bionic quay.io/airshipit/airflow:latest-ubuntu_focal
docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic quay.io/airshipit/shipyard:latest-ubuntu_focal
fi
args:
chdir: "{{ zuul.project.src_dir }}"
become: yes

View File

@ -0,0 +1,101 @@
- hosts: all
roles:
- bindep
- ensure-docker
- ensure-python
- ensure-pip
- ensure-tox
tasks:
- include_vars: vars.yaml
- name: Debug tag generation inputs
block:
- debug:
var: publish
- debug:
var: tags
- debug:
var: distro
- debug:
var: zuul
- debug:
msg: "{{ tags | to_json }}"
- name: Determine tags
shell: echo '{{ tags | to_json }}' | python3 {{ zuul.project.src_dir }}/tools/image_tags.py
environment:
BRANCH: "{{ zuul.branch | default('') }}"
CHANGE: "{{ zuul.change | default('') }}"
COMMIT: "{{ zuul.newrev | default('') }}"
PATCHSET: "{{ zuul.patchset | default('') }}"
register: image_tags
- name: Debug computed tags
debug:
var: image_tags
- name: Install Docker python module for ansible docker login
block:
- pip:
name: docker
version: 4.4.4
executable: pip3
become: True
- name: Make images - verbosive
when: not publish
shell: |
make run_images
args:
chdir: "{{ zuul.project.src_dir }}"
executable: /bin/bash
become: True
- name: Make images
when: not publish
block:
- make:
chdir: "{{ zuul.project.src_dir }}"
target: images
params:
DISTRO: "{{ distro }}"
IMAGE_TAG: "{{ item }}"
with_items: "{{ image_tags.stdout_lines }}"
- shell: "docker images"
register: docker_images
- debug:
var: docker_images
become: True
- name: Publish images
block:
- docker_login:
username: "{{ airship_shipyard_quay_creds.username }}"
password: "{{ airship_shipyard_quay_creds.password }}"
registry_url: "https://quay.io/api/v1/"
- make:
chdir: "{{ zuul.project.src_dir }}"
target: images
params:
DOCKER_REGISTRY: "quay.io"
IMAGE_PREFIX: "airshipit"
DISTRO: "{{ distro }}"
IMAGE_TAG: "{{ item }}"
COMMIT: "{{ zuul.newrev | default('') }}"
PUSH_IMAGE: "true"
with_items: "{{ image_tags.stdout_lines }}"
- shell: "docker images"
register: docker_images
- debug:
var: docker_images
when: publish
become: True

View File

@ -47,6 +47,17 @@
become: True
- name: Make and run images - verbosive
when: not publish
shell: |
make run_images
args:
chdir: "{{ zuul.project.src_dir }}"
executable: /bin/bash
become: True
- name: Make images
when: not publish
block:

21
tox.ini
View File

@ -49,6 +49,27 @@ commands=
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample"
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample"
[testenv:py36]
deps =
tox<=4.0.0
allowlist_externals=
tox
commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py36
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py36
[testenv:cover]
deps =
tox<=4.0.0
allowlist_externals=
tox
sh
commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e cover
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e cover
sh -c 'mv ./src/bin/shipyard_airflow/cover ./'
sh -c 'mv ./src/bin/shipyard_client/cover ./cover/'
[testenv:pep8]
allowlist_externals=
tox