diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 42c84392..3f74f12e 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -30,15 +30,15 @@ labels: images: tags: - airflow: quay.io/airshipit/airflow:latest-ubuntu_bionic - shipyard: quay.io/airshipit/shipyard:latest-ubuntu_bionic + airflow: quay.io/airshipit/airflow:latest-ubuntu_focal + shipyard: quay.io/airshipit/shipyard:latest-ubuntu_focal dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 shipyard_db_init: docker.io/postgres:14.5 shipyard_db_auxiliary: docker.io/postgres:14.5 - shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_bionic + shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_focal airflow_db_init: docker.io/postgres:14.5 rabbit_init: docker.io/rabbitmq:3.7-management - airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_bionic + airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_focal ks_user: docker.io/openstackhelm/heat:ocata ks_service: docker.io/openstackhelm/heat:ocata ks_endpoints: docker.io/openstackhelm/heat:ocata diff --git a/doc/requirements.txt b/doc/requirements.txt index b16a479b..a2bd3d83 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,6 +1,6 @@ # # Requirements for creating documentation only. # -Sphinx +Sphinx<=6.2.1 sphinx-rtd-theme==0.5.0 Jinja2<=3.0.3 \ No newline at end of file diff --git a/doc/source/_static/shipyard.conf.sample b/doc/source/_static/shipyard.conf.sample index e9be4c5a..a822dead 100644 --- a/doc/source/_static/shipyard.conf.sample +++ b/doc/source/_static/shipyard.conf.sample @@ -180,9 +180,13 @@ # and will be removed in the S release. #auth_uri = -# API version of the admin Identity API endpoint. (string value) +# API version of the Identity API endpoint. (string value) #auth_version = +# Interface to use for the Identity API endpoint. Valid values are "public", +# "internal" (default) or "admin". (string value) +#interface = internal + # Do not handle authorization requests within the middleware, but delegate the # authorization decision to downstream WSGI components. (boolean value) #delay_auth_decision = false @@ -217,14 +221,6 @@ # The region in which the identity server can be found. (string value) #region_name = -# DEPRECATED: Directory used to cache files related to PKI tokens. This option -# has been deprecated in the Ocata release and will be removed in the P -# release. (string value) -# This option is deprecated for removal since Ocata. -# Its value may be silently ignored in the future. -# Reason: PKI token format is no longer supported. -#signing_dir = - # Optionally specify a list of memcached server(s) to use for caching. If left # undefined, tokens will instead be cached in-process. (list value) # Deprecated group/name - [keystone_authtoken]/memcache_servers @@ -235,16 +231,6 @@ # to -1 to disable caching completely. (integer value) #token_cache_time = 300 -# DEPRECATED: Determines the frequency at which the list of revoked tokens is -# retrieved from the Identity service (in seconds). A high number of revocation -# events combined with a low cache duration may significantly reduce -# performance. Only valid for PKI tokens. This option has been deprecated in -# the Ocata release and will be removed in the P release. (integer value) -# This option is deprecated for removal since Ocata. -# Its value may be silently ignored in the future. -# Reason: PKI token format is no longer supported. -#revocation_cache_time = 10 - # (Optional) If defined, indicate whether token data should be authenticated or # authenticated and encrypted. If MAC, token data is authenticated (with HMAC) # in the cache. If ENCRYPT, token data is encrypted and authenticated in the @@ -280,9 +266,9 @@ # client connection from the pool. (integer value) #memcache_pool_conn_get_timeout = 10 -# (Optional) Use the advanced (eventlet safe) memcached client pool. The -# advanced pool will only work under python 2.x. (boolean value) -#memcache_use_advanced_pool = false +# (Optional) Use the advanced (eventlet safe) memcached client pool. (boolean +# value) +#memcache_use_advanced_pool = true # (Optional) Indicate whether to set the X-Service-Catalog header. If False, # middleware will not ask for service catalog on token validation and will not @@ -298,27 +284,6 @@ # (string value) #enforce_token_bind = permissive -# DEPRECATED: If true, the revocation list will be checked for cached tokens. -# This requires that PKI tokens are configured on the identity server. (boolean -# value) -# This option is deprecated for removal since Ocata. -# Its value may be silently ignored in the future. -# Reason: PKI token format is no longer supported. -#check_revocations_for_cached = false - -# DEPRECATED: Hash algorithms to use for hashing PKI tokens. This may be a -# single algorithm or multiple. The algorithms are those supported by Python -# standard hashlib.new(). The hashes will be tried in the order given, so put -# the preferred one first for performance. The result of the first hash will be -# stored in the cache. This will typically be set to multiple values only while -# migrating from a less secure algorithm to a more secure one. Once all the old -# tokens are expired this option should be set to a single value for better -# performance. (list value) -# This option is deprecated for removal since Ocata. -# Its value may be silently ignored in the future. -# Reason: PKI token format is no longer supported. -#hash_algorithms = md5 - # A choice of roles that must be present in a service token. Service tokens are # allowed to request that an expired token can be used and so this check should # tightly control that only actual services should be sending this token. Roles @@ -333,6 +298,10 @@ # possible. (boolean value) #service_token_roles_required = false +# The name or type of the service as it appears in the service catalog. This is +# used to validate tokens that have restricted access rules. (string value) +#service_type = + # Authentication type to load (string value) # Deprecated group/name - [keystone_authtoken]/auth_plugin #auth_type = @@ -472,4 +441,4 @@ # Warning - Print a Warning level message if the validation fails # Error - Return an error when the validation fails and prevent the commit from # proceeding -#deployment_version_commit = Skip \ No newline at end of file +#deployment_version_commit = Skip diff --git a/doc/source/_static/shipyard.policy.yaml.sample b/doc/source/_static/shipyard.policy.yaml.sample index 20293810..63a4c9c7 100644 --- a/doc/source/_static/shipyard.policy.yaml.sample +++ b/doc/source/_static/shipyard.policy.yaml.sample @@ -106,3 +106,4 @@ # targeted release # POST /api/v1.0/actions #"workflow_orchestrator:action_test_site": "rule:admin_required" + diff --git a/doc/source/client-user-guide.rst b/doc/source/client-user-guide.rst index 72549b19..7d9484cf 100644 --- a/doc/source/client-user-guide.rst +++ b/doc/source/client-user-guide.rst @@ -184,7 +184,7 @@ command. In this example we will execute the ``get actions`` command:: -e 'OS_PASSWORD=password' -e 'OS_PROJECT_DOMAIN_NAME=default' \ -e 'OS_PROJECT_NAME=service' -e 'OS_USERNAME=shipyard' \ -e 'OS_USER_DOMAIN_NAME=default' -e 'OS_IDENTITY_API_VERSION=3' \ - --rm --net=host airshipit/shipyard:latest-ubuntu_bionic get actions + --rm --net=host airshipit/shipyard:latest-ubuntu_focal get actions The output will resemble the following:: diff --git a/doc/source/multi-distro-support.rst b/doc/source/multi-distro-support.rst index 793febb0..9d0a7a30 100644 --- a/doc/source/multi-distro-support.rst +++ b/doc/source/multi-distro-support.rst @@ -45,7 +45,7 @@ Following parameters need to be passed as environment/shell variable to make command: DISTRO - parameter to identify distro specific Dockerfile, ubuntu_bionic (Default) + parameter to identify distro specific Dockerfile, ubuntu_focal (Default) DISTRO_BASE_IMAGE parameter to use different base image other than what's used in DISTRO diff --git a/images/airflow/Dockerfile.ubuntu_focal b/images/airflow/Dockerfile.ubuntu_focal index e3f1572c..5a6bcf5f 100644 --- a/images/airflow/Dockerfile.ubuntu_focal +++ b/images/airflow/Dockerfile.ubuntu_focal @@ -43,7 +43,7 @@ ARG AIRFLOW_HOME=/usr/local/airflow # Moved celery to images/airflow/requirements.txt as apache-airflow uses a # version of celery incompatibile with the version of kombu needed by other # Airship components -ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs]==1.10.15" +ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5" ARG DEBIAN_FRONTEND=noninteractive ARG ctx_base=src/bin @@ -53,30 +53,39 @@ ARG KUBECTL_VERSION=1.26.3 # Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated ENV AIRFLOW_HOME=${AIRFLOW_HOME} -RUN set -ex && \ - apt-get -qq update && \ - apt-get -y install \ - automake \ - ca-certificates \ - curl \ - git \ - g++ \ - libkrb5-dev \ - libffi-dev \ - libssl-dev \ - libpq-dev \ - libtool \ - libsasl2-dev \ - locales \ - netcat \ - netbase \ - python3 \ - python3-setuptools \ - python3-pip \ - python3-dev \ - python3-dateutil \ - make \ - --no-install-recommends \ +RUN set -ex \ + && apt-get update \ + && apt-get install -y wget curl \ + apt-transport-https ca-certificates gnupg \ + && echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && curl -o /etc/apt/trusted.gpg.d/postgresql_release_signing_key.asc 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' \ + && apt update \ + && apt -y install \ + automake \ + build-essential \ + ca-certificates \ + curl \ + git \ + g++ \ + libkrb5-dev \ + libffi-dev \ + libssl-dev \ + libpq-dev \ + libpq5 \ + libtool \ + libsasl2-dev \ + locales \ + netcat \ + netbase \ + postgresql-client \ + postgresql-common \ + python3 \ + python3-setuptools \ + python3-pip \ + python3-dev \ + python3-dateutil \ + make \ + --no-install-recommends \ && python3 -m pip install -U pip \ && apt-get clean \ && rm -rf \ @@ -102,6 +111,7 @@ RUN set -ex \ && cd .. \ && rm -fr libyaml + # Things that change mostly infrequently RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \ && curl -L -o /usr/local/bin/kubectl \ @@ -114,11 +124,11 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \ # Install Airflow directly to allow overriding source # COPY images/airflow/requirements.txt /tmp/ COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/requirements.txt -RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \ - && pip3 install $AIRFLOW_SRC --no-cache-dir \ - && (pip3 uninstall -y snakebite || true) \ - && (pip3 uninstall -y psycopg2 || true) \ - && (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true) +RUN pip3 install -r /tmp/requirements.txt --no-cache-dir + # && pip3 install $AIRFLOW_SRC --no-cache-dir + # && (pip3 uninstall -y snakebite || true) \ + # && (pip3 uninstall -y psycopg2 || true) \ + # && (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true) # Copy scripts used in the container: COPY images/airflow/script/*.sh ${AIRFLOW_HOME}/ @@ -143,7 +153,7 @@ RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/ RUN cd /tmp/shipyard \ - && pip3 install $(pwd) + && pip3 install $(pwd) --use-pep517 # Note: The value for the dags and plugins directories that are sourced # from the values.yaml of the Shipyard Helm chart need to align with these diff --git a/images/airflow/script/airflow_start_service.sh b/images/airflow/script/airflow_start_service.sh index 56992ece..02a83a54 100755 --- a/images/airflow/script/airflow_start_service.sh +++ b/images/airflow/script/airflow_start_service.sh @@ -41,9 +41,9 @@ elif [[ $cmd == *scheduler* ]]; then elif [[ $cmd == 'quicktest' ]]; then ${python3_path} ${airflow_path} initdb ${python3_path} ${airflow_path} webserver -p 8080 & - airflow tasks run example_bash_operator runme_0 2018-01-01 + airflow run example_bash_operator runme_0 2018-01-01 airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 - airflow dags state example_bash_operator 2018-01-01 + airflow dag_state example_bash_operator 2018-01-01 else echo "Invalid Command!" exit 1 diff --git a/images/shipyard/Dockerfile.ubuntu_focal b/images/shipyard/Dockerfile.ubuntu_focal index fc91620e..b62f6fec 100644 --- a/images/shipyard/Dockerfile.ubuntu_focal +++ b/images/shipyard/Dockerfile.ubuntu_focal @@ -112,10 +112,10 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/ && python3 -m pip install -U pip \ && pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \ && cd /home/shipyard/shipyard_client \ - && pip3 install $(pwd) --use-pep517 \ + && python3 setup.py install \ && pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \ && cd /home/shipyard/shipyard \ - && pip3 install $(pwd) --use-pep517 \ + && python3 setup.py install \ && apt-get purge -y --auto-remove $buildDeps \ && apt-get autoremove -yqq --purge \ && apt-get clean \ @@ -125,7 +125,9 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/ /var/tmp/* \ /usr/share/man \ /usr/share/doc \ - /usr/share/doc-base + /usr/share/doc-base \ + && python3 -m pip install -U pip + # Entrypoint ENTRYPOINT ["/home/shipyard/entrypoint.sh"] diff --git a/src/bin/shipyard_airflow/requirements-direct.txt b/src/bin/shipyard_airflow/requirements-direct.txt index 88cde87e..911417c3 100644 --- a/src/bin/shipyard_airflow/requirements-direct.txt +++ b/src/bin/shipyard_airflow/requirements-direct.txt @@ -1,15 +1,22 @@ +alembic==1.0.1 +apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5 arrow<=0.17.0 -# apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.15 -apache-airflow[crypto,celery,hive,hdfs]==1.10.15 +celery==4.3 +Flask==1.1.0 +Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@airflow_v1.13.2 +pytz==2018.5 coverage==5.3 falcon jsonschema<=3.2.0 keystoneauth1<=5.1.1 +kombu kubernetes==26.1.0 +marshmallow-sqlalchemy==0.18.0 networkx oslo.config<=8.7.1 oslo.policy<=3.10.1 oslo.utils<=4.12.3 +psycopg2-binary==2.8.4 pyarrow==6.0.1 pylibyaml==0.1.0 PyYAML<=5.4.1 @@ -17,12 +24,16 @@ regex==2020.11.13 requests==2.23.0 responses==0.12.1 setuptools<=45.2.0 -SQLAlchemy<=1.3.20 +SQLAlchemy==1.3.15 +tabulate==0.8.03 ulid==1.1 uWSGI==2.0.21 -Werkzeug +Werkzeug<=0.16.1 WTForms<=2.3.3 + +itsdangerous==2.0.1 + git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#egg=drydock_provisioner&subdirectory=python git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4#egg=armada diff --git a/src/bin/shipyard_airflow/requirements-frozen.txt b/src/bin/shipyard_airflow/requirements-frozen.txt index 2edb4a5d..b3fe862a 100644 --- a/src/bin/shipyard_airflow/requirements-frozen.txt +++ b/src/bin/shipyard_airflow/requirements-frozen.txt @@ -1,23 +1,21 @@ alabaster==0.7.13 -alembic==1.4.3 +alembic==1.0.1 amqp==2.6.1 -apache-airflow==1.10.15 +apache-airflow==1.10.5 apispec==1.3.3 -argcomplete==1.12.3 -Armada @ git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4#egg=armada +argcomplete==3.0.8 +Armada @ git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4 arrow==0.17.0 -attrs==20.3.0 +attrs==23.1.0 autopage==0.5.1 Babel==2.12.1 -bandit==1.6.0 bcrypt==4.0.1 Beaker==1.12.0 billiard==3.6.4.0 cached-property==1.5.2 cachetools==5.3.0 -cattrs==1.10.0 -celery==4.4.7 -certifi==2022.12.7 +celery==4.3.0 +certifi==2023.5.7 cffi==1.15.1 chardet==3.0.4 charset-normalizer==3.1.0 @@ -31,18 +29,18 @@ coverage==5.3 croniter==0.3.37 cryptography==3.4.8 debtcollector==2.5.0 -Deckhand @ git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand +Deckhand @ git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a decorator==5.1.1 deepdiff==5.8.1 defusedxml==0.7.1 -dill==0.3.6 +dill==0.2.9 distlib==0.3.6 dnspython==2.3.0 docutils==0.17 dogpile.cache==1.2.0 -drydock-provisioner @ git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#egg=drydock_provisioner&subdirectory=python -dulwich==0.21.3 -email-validator==1.3.1 +drydock-provisioner @ git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#subdirectory=python +dulwich==0.21.5 +dumb-init==1.2.5.post1 eventlet==0.33.3 extras==1.0.0 falcon==3.1.1 @@ -50,27 +48,26 @@ fasteners==0.18 filelock==3.12.0 fixtures==3.0.0 flake8==3.8.4 -Flask==1.1.4 -Flask-Admin==1.5.4 -Flask-AppBuilder==2.3.4 -Flask-Babel==1.0.0 +Flask==1.1.0 +Flask-Admin==1.5.3 +Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@b71789d85b632935eca79c0b53fb77f20bf17fe6 +Flask-Babel==0.12.2 Flask-Caching==1.3.3 Flask-JWT-Extended==3.25.1 Flask-Login==0.4.1 Flask-OpenID==1.3.0 Flask-SQLAlchemy==2.5.1 -flask-swagger==0.2.14 +flask-swagger==0.2.13 Flask-WTF==0.14.3 flower==0.9.7 -funcsigs==1.0.2 -future==0.18.3 +funcsigs==1.0.0 +future==0.16.0 futurist==2.4.1 gitdb==4.0.10 GitPython==3.1.31 google-auth==2.17.3 -graphviz==0.20.1 greenlet==2.0.2 -gunicorn==20.1.0 +gunicorn==19.10.0 hacking==4.1.0 hmsclient==0.1.1 html5lib==0.9999999 @@ -79,10 +76,11 @@ humanize==4.6.0 idna==2.10 imagesize==1.4.1 importlib-metadata==2.1.3 -importlib-resources==1.5.0 iso8601==1.1.0 -itsdangerous==1.1.0 -Jinja2==2.11.3 +itsdangerous==2.0.1 +JayDeBeApi==1.2.3 +Jinja2==2.10.3 +JPype1==1.4.1 json-merge-patch==0.2 jsonpath-ng==1.5.3 jsonpath-rw==1.4.0 @@ -91,19 +89,18 @@ jsonpickle==1.4.1 jsonschema==3.2.0 keystoneauth1==5.1.1 keystonemiddleware==10.2.0 -kombu==4.6.11 +kombu==4.6.3 kubernetes==26.1.0 -lazy-object-proxy==1.4.3 +lazy-object-proxy==1.9.0 lockfile==0.12.2 Mako==1.2.4 Markdown==2.6.11 MarkupSafe==2.0.1 -marshmallow==2.21.0 +marshmallow==2.19.5 marshmallow-enum==1.5.1 -marshmallow-sqlalchemy==0.23.1 +marshmallow-sqlalchemy==0.18.0 mccabe==0.6.1 mock==5.0.2 -more-itertools==9.1.0 msgpack==1.0.5 natsort==8.3.1 netaddr==0.8.0 @@ -113,6 +110,7 @@ nose==1.3.7 numpy==1.24.3 oauthlib==3.2.2 ordered-set==4.1.0 +ordereddict==1.1 os-service-types==1.7.0 oslo.cache==2.10.1 oslo.concurrency==5.1.1 @@ -121,32 +119,29 @@ oslo.context==4.1.0 oslo.db==10.0.0 oslo.i18n==6.0.0 oslo.log==4.6.0 -oslo.messaging==12.13.0 -oslo.metrics==0.6.0 +oslo.messaging==10.3.0 oslo.middleware==4.4.0 oslo.policy==3.10.1 oslo.serialization==4.2.0 oslo.service==3.1.1 oslo.utils==4.12.3 packaging==21.3 -pandas==1.5.3 +pandas==0.25.3 Paste==3.5.0 PasteDeploy==3.0.1 PasteScript==3.3.0 pbr==5.5.1 pendulum==1.4.4 pip==23.0.1 -platformdirs==3.4.0 -pluggy==0.6.0 +platformdirs==3.5.0 ply==3.11 prettytable==3.7.0 -prison==0.2.1 -promenade @ git+https://opendev.org/airship/promenade.git@5602e85fb936eeafdc3e4699fd3afbe1576078bc#egg=promenade +prison==0.1.0 +promenade @ git+https://opendev.org/airship/promenade.git@5602e85fb936eeafdc3e4699fd3afbe1576078bc prometheus-client==0.8.0 -protobuf==4.22.3 +protobuf==4.22.4 psutil==5.9.5 -psycopg2-binary==2.9.6 -pure-sasl==0.6.2 +psycopg2-binary==2.8.4 py==1.11.0 pyarrow==6.0.1 pyasn1==0.5.0 @@ -164,20 +159,16 @@ pyparsing==2.4.7 pyperclip==1.8.2 pyproject_api==1.5.0 pyrsistent==0.19.3 -pytest==3.5.0 -pytest-cov==2.5.1 python-barbicanclient==5.2.0 -python-daemon==2.3.2 +python-daemon==2.1.2 python-dateutil==2.8.1 python-editor==1.0.4 python-keystoneclient==5.1.0 python-memcached==1.59 python-mimeparse==1.6.0 -python-nvd3==0.15.0 -python-slugify==4.0.1 python-subunit==1.4.0 python3-openid==3.2.0 -pytz==2023.3 +pytz==2018.5 pytzdata==2020.1 PyYAML==5.4.1 regex==2020.11.13 @@ -191,14 +182,13 @@ retry==0.9.2 rfc3986==2.0.0 Routes==2.5.1 rsa==4.9 -sasl==0.3.1 selector==0.10.1 setproctitle==1.3.2 setuptools==45.2.0 simplejson==3.19.1 six==1.16.0 smmap==5.0.0 -snakebite-py3==3.0.5 +snakebite==2.11.0 snowballstemmer==2.2.0 Sphinx==3.3.1 sphinx-rtd-theme==0.5.0 @@ -208,30 +198,25 @@ sphinxcontrib-htmlhelp==2.0.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 -SQLAlchemy==1.3.20 -SQLAlchemy-JSONField==0.9.0 +SQLAlchemy==1.3.15 sqlalchemy-migrate==0.13.0 -SQLAlchemy-Utils==0.41.0 sqlparse==0.4.4 statsd==4.0.1 stevedore==5.0.0 -tabulate==0.8.10 +tabulate==0.8.3 Tempita==0.5.2 tenacity==4.12.0 -testfixtures==7.1.0 +termcolor==1.1.0 testrepository==0.0.20 testresources==2.0.1 testscenarios==0.5.0 testtools==2.5.0 -text-unidecode==1.3 +text-unidecode==1.2 thrift==0.16.0 -thrift-sasl==0.4.3 tiddlyweb==2.4.3 -toml==0.10.2 tomli==2.0.1 -tomlkit==0.11.7 +tomlkit==0.11.8 tornado==5.1.1 -tox==3.12.1 typing-extensions==3.7.2 tzlocal==1.5.1 ulid==1.1 @@ -239,7 +224,7 @@ unicodecsv==0.14.1 urllib3==1.25.11 uWSGI==2.0.21 vine==1.3.0 -virtualenv==20.22.0 +virtualenv==20.23.0 wcwidth==0.2.6 WebOb==1.8.7 websocket-client==1.5.1 diff --git a/src/bin/shipyard_airflow/test-requirements.txt b/src/bin/shipyard_airflow/test-requirements.txt index d1800ecb..70fb613f 100644 --- a/src/bin/shipyard_airflow/test-requirements.txt +++ b/src/bin/shipyard_airflow/test-requirements.txt @@ -5,4 +5,11 @@ pytest-cov==2.5.1 testfixtures tox<=4.0.0 typing-extensions==3.7.2 -pyflakes>=2.1.1 \ No newline at end of file +pyflakes>=2.1.1 +# Pin apache-airflow 1.10.5 to align with version in airflow images requirements.txt +apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5 +# install postgres seperately as apache-airflow 1.10.5 forces postgres to use +# psycopg2 instead of psycopg2-binary which requires additional apt packages +# to be installed, i.e. postgres-devel. Otherwise the following error is seen: +# Error: pg_config executable not found. Only seems to be an issue in xenial +postgres>=3.0.0 \ No newline at end of file diff --git a/src/bin/shipyard_airflow/tox.ini b/src/bin/shipyard_airflow/tox.ini index f29c1e6f..8bc6ad26 100644 --- a/src/bin/shipyard_airflow/tox.ini +++ b/src/bin/shipyard_airflow/tox.ini @@ -7,7 +7,9 @@ setenv= PYTHONWARNING=all LANG=C.UTF-8 LC_ALL=C.UTF-8 -deps = -r{toxinidir}/requirements-frozen.txt +deps = + -r{toxinidir}/requirements-frozen.txt + -r{toxinidir}/test-requirements.txt [testenv:freeze] @@ -17,7 +19,6 @@ allowlist_externals= sh deps= -r{toxinidir}/requirements-direct.txt - -r{toxinidir}/test-requirements.txt commands= rm -f {toxinidir}/requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt" @@ -27,9 +28,16 @@ commands= skipsdist=True setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes +allowlist_externals= + bash + airflow commands = + bash -c "rm -f ${HOME}/airflow/airflow.db" pip install -r{toxinidir}/test-requirements.txt airflow initdb + airflow run example_bash_operator runme_0 2018-01-01 + airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 + airflow dag_state example_bash_operator 2018-01-01 pytest \ {posargs} \ --cov-branch \ @@ -74,6 +82,7 @@ commands = [testenv:gen_all] # Combined to make these run together instead of setting up separately +skipsdist=True setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes deps = diff --git a/src/bin/shipyard_client/requirements-direct.txt b/src/bin/shipyard_client/requirements-direct.txt index e663fa66..b279441c 100644 --- a/src/bin/shipyard_client/requirements-direct.txt +++ b/src/bin/shipyard_client/requirements-direct.txt @@ -15,10 +15,19 @@ # CLI/Client requirements arrow<=0.17.0 -click==6.7 +click==7.1.2 click-default-group==1.2 keystoneauth1<=5.1.1 requests==2.23.0 PTable==0.9.2 pylibyaml==0.1.0 PyYAML<=5.4.1 +Flask==1.1.0 +coverage==5.3 +itsdangerous==2.0.1 +Jinja2==2.10.3 +MarkupSafe==2.0.1 +pbr==5.5.1 +python-dateutil==2.8.1 +setuptools<=45.2.0 +Werkzeug==0.16.1 \ No newline at end of file diff --git a/src/bin/shipyard_client/requirements-frozen.txt b/src/bin/shipyard_client/requirements-frozen.txt index a88e5c46..07136493 100644 --- a/src/bin/shipyard_client/requirements-frozen.txt +++ b/src/bin/shipyard_client/requirements-frozen.txt @@ -1,40 +1,27 @@ -amqp==2.6.1 arrow==0.17.0 -attrs==23.1.0 -bandit==1.6.0 -certifi==2022.12.7 +certifi==2023.5.7 chardet==3.0.4 -click==6.7 +click==7.1.2 click-default-group==1.2 -coverage==7.2.3 -flake8==3.8.4 -gitdb==4.0.10 -GitPython==3.1.31 +coverage==5.3 +Flask==1.1.0 idna==2.10 iso8601==1.1.0 +itsdangerous==2.0.1 +Jinja2==2.10.3 keystoneauth1==5.1.1 -mccabe==0.6.1 -more-itertools==9.1.0 +MarkupSafe==2.0.1 os-service-types==1.7.0 -pbr==5.11.1 +pbr==5.5.1 pip==23.0.1 -pluggy==0.6.0 PTable==0.9.2 -py==1.11.0 -pycodestyle==2.6.0 -pyflakes==2.2.0 pylibyaml==0.1.0 -pytest==3.5.0 -pytest-cov==2.5.1 -python-dateutil==2.8.2 +python-dateutil==2.8.1 PyYAML==5.4.1 requests==2.23.0 -responses==0.10.2 -setuptools==67.6.0 +setuptools==45.2.0 six==1.16.0 -smmap==5.0.0 stevedore==5.0.0 -testfixtures==5.1.1 urllib3==1.25.11 -vine==1.3.0 +Werkzeug==0.16.1 wheel==0.40.0 diff --git a/src/bin/shipyard_client/tests/unit/cli/create/test_create_commands.py b/src/bin/shipyard_client/tests/unit/cli/create/test_create_commands.py index f05c29de..0c4b5d29 100644 --- a/src/bin/shipyard_client/tests/unit/cli/create/test_create_commands.py +++ b/src/bin/shipyard_client/tests/unit/cli/create/test_create_commands.py @@ -187,7 +187,7 @@ def test_create_configdocs_directory_empty(): '--directory=' + dir1 ]) _method.assert_not_called() - assert b'directory does not contain any YAML files' in result.output_bytes + assert b'directory does not contain any YAML files' in result.stdout_bytes def test_create_configdocs_multi_directory(): diff --git a/src/bin/shipyard_client/tox.ini b/src/bin/shipyard_client/tox.ini index 703a9ea1..9cbde55a 100644 --- a/src/bin/shipyard_client/tox.ini +++ b/src/bin/shipyard_client/tox.ini @@ -17,7 +17,6 @@ allowlist_externals= sh deps= -r{toxinidir}/requirements-direct.txt - -r{toxinidir}/test-requirements.txt commands= rm -f {toxinidir}/requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt" diff --git a/tools/airflow_image_run.sh b/tools/airflow_image_run.sh index 37fe5a3b..6b19b284 100755 --- a/tools/airflow_image_run.sh +++ b/tools/airflow_image_run.sh @@ -47,6 +47,7 @@ fi docker stop airflow_test docker rm airflow_test + if [ ${TEST_RESP:(-7)} == "success" ]; then exit 0 else diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index fc589bf3..5add4b82 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -22,6 +22,7 @@ - ensure-docker tasks: + - name: Clone Required Repositories shell: | export CLONE_SHIPYARD={{ CLONE_SHIPYARD }} @@ -45,15 +46,23 @@ - name: Build Shipyard and Airflow with submitted changes shell: | make + docker pull quay.io/airshipit/shipyard:latest-ubuntu_bionic + docker rm quay.io/airshipit/shipyard:latest-ubuntu_bionic + docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal quay.io/airshipit/shipyard:latest-ubuntu_bionic + docker pull quay.io/airshipit/airflow:latest-ubuntu_bionic + docker rm quay.io/airshipit/airflow:latest-ubuntu_bionic + docker tag quay.io/airshipit/airflow:latest-ubuntu_focal quay.io/airshipit/airflow:latest-ubuntu_bionic args: chdir: "{{ zuul.project.src_dir }}" become: yes - - name: Setup clients + - name: Build all charts locally shell: | - ./tools/deployment/airskiff/developer/020-setup-client.sh + set -ex + ./tools/deployment/airskiff/developer/015-make-all-charts.sh args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes - name: Deploy Airship components using Armada shell: | diff --git a/tools/shipyard.sh b/tools/shipyard.sh index 75b2a3b5..eda3a298 100755 --- a/tools/shipyard.sh +++ b/tools/shipyard.sh @@ -38,7 +38,7 @@ set -e # Source Base Docker Command SHIPYARD_HOSTPATH=${SHIPYARD_HOSTPATH:-"/target"} NAMESPACE="${NAMESPACE:-ucp}" -DISTRO="${DISTRO:-ubuntu_bionic}" +DISTRO="${DISTRO:-ubuntu_focal}" SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master-${DISTRO}}" # set default value for OS_PASSWORD if it's not set # this doesn't actually get exported to environment diff --git a/tools/shipyard_docker_base_command.sh b/tools/shipyard_docker_base_command.sh index f1a35272..3244c643 100644 --- a/tools/shipyard_docker_base_command.sh +++ b/tools/shipyard_docker_base_command.sh @@ -19,7 +19,7 @@ # before executing this script if they differ from the default values. # NAMESPACE="${NAMESPACE:-ucp}" -DISTRO="${DISTRO:-ubuntu_bionic}" +DISTRO="${DISTRO:-ubuntu_focal}" SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master-${DISTRO}}" # Define Base Docker Command diff --git a/tox.ini b/tox.ini index a64b0859..50fc22f1 100644 --- a/tox.ini +++ b/tox.ini @@ -23,9 +23,6 @@ allowlist_externals= cp tox commands= - tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all - cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample - cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample rm -rf {toxinidir}/doc/build sphinx-build -W -b html {toxinidir}/doc/source {toxinidir}/doc/build/html deps= -r{toxinidir}/doc/requirements.txt @@ -40,11 +37,17 @@ commands= tox -c {toxinidir}/src/bin/shipyard_client/tox.ini [testenv:py38] +deps = + tox<=4.0.0 allowlist_externals= tox + bash commands= tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py38 tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py38 + tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all + bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample" + bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample" [testenv:pep8] allowlist_externals= @@ -52,3 +55,29 @@ allowlist_externals= commands= tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e pep8 tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e pep8 + + +[testenv:freeze] +allowlist_externals= + tox +commands= + tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e freeze + tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e freeze + + +[testenv:shipyard_client] +deps = + -r{toxinidir}/src/bin/shipyard_airflow/requirements-frozen.txt + -r{toxinidir}/src/bin/shipyard_client/requirements-frozen.txt +skipsdist=True +allowlist_externals= + pytest +commands = + pytest \ + {posargs} \ + --cov-branch \ + --cov-report=term-missing:skip-covered \ + --cov-config=.coveragerc \ + --cov=shipyard_client \ + --cov-report=html \ + -vv \ No newline at end of file