Airflow stable 2.8.1

Change-Id: Iac800d9bc6e53a5a60e11c649d05dc663e8dd8e9
This commit is contained in:
Sergiy Markin 2024-02-21 21:07:39 +00:00
parent 84cd4e524d
commit 20593bc746
12 changed files with 212 additions and 190 deletions

View File

@ -72,7 +72,7 @@
run: tools/gate/playbooks/build-charts.yaml
nodeset: airship-shipyard-single-node-focal
vars:
HTK_COMMIT: ae91cf3fc3f288b6d92ace4a3a405606a653638f
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
- job:
name: airship-shipyard-chart-build-latest-htk
@ -133,10 +133,10 @@
vars:
CLONE_SHIPYARD: false
SHIPYARD_IMAGE_DISTRO: ubuntu_focal
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz
HTK_COMMIT: ae91cf3fc3f288b6d92ace4a3a405606a653638f
OSH_INFRA_COMMIT: db3537e56b182a54e7f6931ce57e2a190714019b
OSH_COMMIT: 75c30f43db44218e7842611e880fd8d7a30fa79c
HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
irrelevant-files: *irrelevant-files
@ -157,7 +157,7 @@
commit: true
static:
- latest
- airflow_2.6.2
- airflow_2.8.1

View File

@ -16,8 +16,8 @@
apiVersion: v1
description: A Helm chart for Shipyard and Airflow
name: shipyard
version: 0.2.3
appVersion: 2.6.2
version: 0.2.4
appVersion: 2.8.1
keywords:
- shipyard
- airflow

View File

@ -19,4 +19,5 @@ limitations under the License.
set -ex
airflow db upgrade
airflow db init
airflow db migrate

View File

@ -688,7 +688,7 @@ conf:
api_client: airflow.api.client.local_client
# if endpoint_url is not set, it is extracted from endpoints by the
# configmap template
endpoint_url: http://localhost/
endpoint_url: http://localhost
api:
auth_backend: airflow.api.auth.backend.default
enable_experimental_api: "True"
@ -744,7 +744,7 @@ conf:
webserver:
# if base_url is not set, is extracted from endpoints by the configmap
# template
base_url: http://localhost/
base_url: http://localhost
# set web_server_host to 0.0.0.0 to bind to all interfaces. By default
# only bind to loopback
web_server_host: 127.0.0.1
@ -787,7 +787,7 @@ conf:
smtp_port: 25
smtp_mail_from: airflow@airflow.local
celery:
celery_app_name: airflow.executors.celery_executor
celery_app_name: airflow.providers.celery.executors.celery_executor
# The concurrency that will be used when starting workers with the
# "airflow worker" command. This defines the number of task instances
# that a worker will take, so size up your workers based on the resources

View File

@ -44,12 +44,12 @@ ARG AIRFLOW_HOME=/usr/local/airflow
# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
# version of celery incompatibile with the version of kombu needed by other
# Airship components
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.6.2"
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.8.1"
ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin
# Kubectl version
ARG KUBECTL_VERSION=1.27.1
ARG KUBECTL_VERSION=1.29.0
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
ENV AIRFLOW_HOME=${AIRFLOW_HOME}

View File

@ -21,6 +21,7 @@ airflow_path=$(which airflow)
# Initialize Airflow DB
if [[ $cmd == 'db init' ]]; then
${python3_path} ${airflow_path} db init
${python3_path} ${airflow_path} db migrate
# Start the services based on argument from Airflow Helm Chart
elif [[ $cmd == 'webserver' ]]; then
${python3_path} ${airflow_path} webserver
@ -40,7 +41,7 @@ elif [[ $cmd == *scheduler* ]]; then
done
elif [[ $cmd == 'quicktest' ]]; then
${python3_path} ${airflow_path} db init
${python3_path} ${airflow_path} db upgrade
${python3_path} ${airflow_path} db migrate
${python3_path} ${airflow_path} dags list
${python3_path} ${airflow_path} webserver -p 8080 &
${python3_path} ${airflow_path} tasks test example_bash_operator runme_0

View File

@ -1,9 +1,10 @@
alembic
apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.6.2
apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.8.1
arrow
celery
Flask
Flask-AppBuilder
graphviz
pytz
falcon
jsonschema
@ -49,7 +50,7 @@ WTForms
itsdangerous
git+https://opendev.org/airship/deckhand.git@37ba567da5bf7f146f54a468478a025c70793574#egg=deckhand
git+https://opendev.org/airship/drydock.git@f99abfa4337f8cbb591513aac404b11208d4187c#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@6595dd83ea65324196c89cf6fb83f168818822de#egg=armada
git+https://opendev.org/airship/promenade.git@69a74590e76e810916f7780fc525c63ec58c7dc1#egg=promenade
git+https://opendev.org/airship/deckhand.git@5d2765e513fa55be74aef4456d6b353d24bc7097#egg=deckhand
git+https://opendev.org/airship/drydock.git@1e180a3f51fcb1d00c85e5691fde548b934d0572#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@b3b8d05fe71cf42a3af0cd1bc50110ef989024a9#egg=armada
git+https://opendev.org/airship/promenade.git@1a0ea41f286980bf1ccdb185578d9795983512eb#egg=promenade

View File

@ -1,144 +1,158 @@
aiohttp==3.8.4
aiohttp==3.9.1
aiosignal==1.3.1
alembic==1.11.1
amqp==5.1.1
anyio==3.7.0
apache-airflow==2.6.2
apache-airflow-providers-apache-hdfs==4.0.0
apache-airflow-providers-apache-hive==6.1.0
apache-airflow-providers-celery==3.2.0
apache-airflow-providers-common-sql==1.5.1
apache-airflow-providers-ftp==3.4.1
apache-airflow-providers-http==4.4.1
apache-airflow-providers-imap==3.2.1
apache-airflow-providers-jdbc==3.4.0
apache-airflow-providers-postgres==5.5.0
apache-airflow-providers-sqlite==3.4.1
apispec==5.2.2
argcomplete==3.1.1
Armada @ git+https://opendev.org/airship/armada.git@6595dd83ea65324196c89cf6fb83f168818822de
arrow==1.2.3
alembic==1.13.1
amqp==5.2.0
annotated-types==0.6.0
anyio==4.2.0
apache-airflow==2.8.1
apache-airflow-providers-apache-hdfs==4.3.2
apache-airflow-providers-apache-hive==6.4.1
apache-airflow-providers-celery==3.5.1
apache-airflow-providers-common-io==1.2.0
apache-airflow-providers-common-sql==1.10.0
apache-airflow-providers-ftp==3.7.0
apache-airflow-providers-http==4.8.0
apache-airflow-providers-imap==3.5.0
apache-airflow-providers-jdbc==4.2.1
apache-airflow-providers-postgres==5.10.0
apache-airflow-providers-sqlite==3.7.0
apispec==6.4.0
argcomplete==3.2.1
Armada @ git+https://opendev.org/airship/armada.git@b3b8d05fe71cf42a3af0cd1bc50110ef989024a9
arrow==1.3.0
asgiref==3.7.2
async-timeout==4.0.2
attrs==23.1.0
autopage==0.5.1
Babel==2.12.1
async-timeout==4.0.3
attrs==23.2.0
autopage==0.5.2
Babel==2.14.0
backoff==2.2.1
backports.zoneinfo==0.2.1
barbican==14.0.2
bcrypt==4.0.1
bcrypt==4.1.2
Beaker==1.12.1
billiard==4.1.0
blinker==1.6.2
billiard==4.2.0
blinker==1.7.0
cachelib==0.9.0
cachetools==5.3.1
castellan==4.1.0
cattrs==23.1.2
celery==5.3.0
certifi==2023.5.7
cffi==1.15.1
charset-normalizer==3.1.0
click==8.1.3
cachetools==5.3.2
castellan==4.3.0
celery==5.3.6
certifi==2023.11.17
cffi==1.16.0
charset-normalizer==3.3.2
click==8.1.7
click-didyoumean==0.3.0
click-plugins==1.1.1
click-repl==0.2.0
click-repl==0.3.0
clickclick==20.10.2
cliff==4.3.0
cliff==4.6.0
cmd2==2.4.3
colorama==0.4.6
colorlog==4.8.0
ConfigUpdater==3.1.1
ConfigUpdater==3.2
connexion==2.14.2
cron-descriptor==1.4.0
croniter==1.3.15
cryptography==40.0.2
debtcollector==2.5.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@37ba567da5bf7f146f54a468478a025c70793574
croniter==2.0.1
cryptography==41.0.7
debtcollector==3.0.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@5d2765e513fa55be74aef4456d6b353d24bc7097
decorator==5.1.1
deepdiff==6.3.1
deepdiff==6.7.1
Deprecated==1.2.14
dill==0.3.1.1
dnspython==2.3.0
dnspython==2.4.2
docopt==0.6.2
docutils==0.20.1
dogpile.cache==1.2.2
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@f99abfa4337f8cbb591513aac404b11208d4187c#subdirectory=python
dogpile.cache==1.3.2
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@1e180a3f51fcb1d00c85e5691fde548b934d0572#subdirectory=python
email-validator==1.3.1
eventlet==0.33.3
exceptiongroup==1.1.1
falcon==3.1.1
fastavro==1.7.4
fasteners==0.18
eventlet==0.34.3
exceptiongroup==1.2.0
falcon==3.1.3
fastavro==1.9.3
fasteners==0.19
fixtures==4.1.0
Flask==2.2.5
Flask-AppBuilder==4.3.1
Flask-AppBuilder==4.3.10
Flask-Babel==2.0.0
Flask-Caching==2.0.2
Flask-JWT-Extended==4.5.2
Flask-Limiter==3.3.1
Flask-Login==0.6.2
Flask-Caching==2.1.0
Flask-JWT-Extended==4.6.0
Flask-Limiter==3.5.0
Flask-Login==0.6.3
Flask-Session==0.5.0
Flask-SQLAlchemy==2.5.1
Flask-WTF==1.1.1
flower==1.2.0
frozenlist==1.3.3
Flask-WTF==1.2.1
flower==2.0.1
frozenlist==1.4.1
fsspec==2023.12.2
future==0.18.3
futurist==2.4.1
gitdb==4.0.10
GitPython==3.1.31
google-auth==2.20.0
futurist==3.0.0
gitdb==4.0.11
GitPython==3.1.41
google-auth==2.26.2
google-re2==1.1
googleapis-common-protos==1.62.0
graphviz==0.20.1
greenlet==2.0.2
gssapi==1.8.2
gunicorn==20.1.0
greenlet==3.0.3
grpcio==1.60.0
gssapi==1.8.3
gunicorn==21.2.0
h11==0.14.0
hdfs==2.7.0
hdfs==2.7.3
hmsclient==0.1.1
html5lib==0.9999999
httpcore==0.16.3
httpexceptor==1.4.0
httpx==0.23.3
humanize==4.6.0
idna==3.4
importlib-metadata==4.13.0
importlib-resources==5.12.0
humanize==4.9.0
idna==3.6
importlib-metadata==6.11.0
importlib-resources==5.13.0
inflection==0.5.1
iso8601==2.0.0
iso8601==2.1.0
itsdangerous==2.1.2
JayDeBeApi==1.2.3
Jinja2==3.1.2
JPype1==1.4.1
jsonpath-ng==1.5.3
jsonpickle==3.0.1
jsonschema==4.17.3
Jinja2==3.1.3
JPype1==1.5.0
jsonpath-ng==1.6.1
jsonpickle==3.0.2
jsonschema==4.20.0
jsonschema-specifications==2023.12.1
keystoneauth1==4.3.1
keystonemiddleware==9.3.0
kombu==5.3.0
krb5==0.5.0
kombu==5.3.5
krb5==0.5.1
kubernetes==23.6.0
lazy-object-proxy==1.9.0
lazy-object-proxy==1.10.0
ldap3==2.9.1
limits==3.5.0
limits==3.7.0
linkify-it-py==2.0.2
lockfile==0.12.2
logutils==0.3.5
Mako==1.2.4
Markdown==3.4.3
Mako==1.3.0
Markdown==3.5.2
markdown-it-py==3.0.0
MarkupSafe==2.1.3
marshmallow==3.19.0
marshmallow-enum==1.5.1
marshmallow==3.20.2
marshmallow-oneofschema==3.0.1
marshmallow-sqlalchemy==0.26.1
mdit-py-plugins==0.4.0
mdurl==0.1.2
mock==5.1.0
msgpack==1.0.5
msgpack==1.0.7
multidict==6.0.4
netaddr==0.8.0
netaddr==1.2.1
netifaces==0.11.0
networkx==3.1
numpy==1.24.3
numpy==1.24.4
oauthlib==3.2.2
opentelemetry-api==1.22.0
opentelemetry-exporter-otlp==1.22.0
opentelemetry-exporter-otlp-proto-common==1.22.0
opentelemetry-exporter-otlp-proto-grpc==1.22.0
opentelemetry-exporter-otlp-proto-http==1.22.0
opentelemetry-proto==1.22.0
opentelemetry-sdk==1.22.0
opentelemetry-semantic-conventions==0.43b0
ordered-set==4.1.0
os-service-types==1.7.0
oslo.cache==2.10.0
@ -157,113 +171,117 @@ oslo.service==2.8.0
oslo.upgradecheck==2.1.1
oslo.utils==4.12.3
oslo.versionedobjects==2.4.0
packaging==21.3
pandas==1.5.3
Paste==3.5.3
PasteDeploy==3.0.1
PasteScript==3.3.0
pathspec==0.9.0
pbr==5.11.1
packaging==23.2
pandas==2.0.3
Paste==3.7.1
PasteDeploy==3.1.0
PasteScript==3.4.0
pathspec==0.12.1
pbr==6.0.0
pecan==1.5.1
pendulum==2.1.2
pip==23.1.2
pendulum==3.0.0
pip==23.2.1
pkgutil_resolve_name==1.3.10
pluggy==1.0.0
pluggy==1.3.0
ply==3.11
prettytable==3.8.0
prettytable==3.10.0
prison==0.2.1
promenade @ git+https://opendev.org/airship/promenade.git@69a74590e76e810916f7780fc525c63ec58c7dc1
prometheus-client==0.17.0
prompt-toolkit==3.0.38
psutil==5.9.5
psycopg2-binary==2.9.6
promenade @ git+https://opendev.org/airship/promenade.git@1a0ea41f286980bf1ccdb185578d9795983512eb
prometheus-client==0.19.0
prompt-toolkit==3.0.43
protobuf==4.25.2
psutil==5.9.7
psycopg2-binary==2.9.9
pure-sasl==0.6.2
py==1.11.0
pyarrow==11.0.0
pyasn1==0.4.8
pyasn1-modules==0.2.8
pyarrow==14.0.2
pyasn1==0.5.1
pyasn1-modules==0.3.0
pycadf==3.1.1
pycparser==2.21
pydantic==1.10.9
Pygments==2.15.1
PyHive==0.6.5
PyJWT==2.7.0
pydantic==2.5.3
pydantic_core==2.14.6
Pygments==2.17.2
PyHive==0.7.0
PyJWT==2.8.0
pylibyaml==0.1.0
pyOpenSSL==23.2.0
pyparsing==3.0.9
pyOpenSSL==23.3.0
pyparsing==3.1.1
pyperclip==1.8.2
pyrsistent==0.19.3
pyspnego==0.9.1
pyspnego==0.10.2
python-barbicanclient==5.4.0
python-daemon==3.0.1
python-dateutil==2.8.2
python-keystoneclient==5.1.0
python-memcached==1.59
python-keystoneclient==5.3.0
python-memcached==1.62
python-mimeparse==1.6.0
python-nvd3==0.15.0
python-slugify==8.0.1
python3-memcached==1.51
pytz==2023.3
pytzdata==2020.1
PyYAML==6.0
regex==2023.6.3
pytz==2023.3.post1
PyYAML==6.0.1
referencing==0.32.1
regex==2023.12.25
repoze.lru==0.7
requests==2.31.0
requests-kerberos==0.14.0
requests-oauthlib==1.3.1
requests-toolbelt==1.0.0
resolver==0.2.1
responses==0.23.1
responses==0.24.1
retry==0.9.2
rfc3339-validator==0.1.4
rfc3986==1.5.0
rich==13.4.2
rich_argparse==1.1.1
rich==13.7.0
rich-argparse==1.4.0
Routes==2.5.1
rpds-py==0.17.1
rsa==4.9
sasl==0.3.1
selector==0.10.1
setproctitle==1.3.2
setuptools==67.7.2
simplejson==3.19.1
setproctitle==1.3.3
setuptools==68.2.2
simplejson==3.19.2
six==1.16.0
smmap==5.0.0
smmap==5.0.1
sniffio==1.3.0
SQLAlchemy==1.4.48
SQLAlchemy-JSONField==1.0.1.post0
SQLAlchemy==1.4.51
SQLAlchemy-JSONField==1.0.2
sqlalchemy-migrate==0.13.0
SQLAlchemy-Utils==0.41.1
sqlparse==0.4.4
statsd==4.0.1
stevedore==5.1.0
stevedore==5.2.0
tabulate==0.9.0
Tempita==0.5.2
tenacity==8.2.2
termcolor==2.3.0
tenacity==8.2.3
termcolor==2.4.0
testresources==2.0.1
testscenarios==0.5.0
testtools==2.6.0
testtools==2.7.1
text-unidecode==1.3
thrift==0.16.0
thrift-sasl==0.4.3
tiddlyweb==2.4.3
tornado==6.3.2
types-PyYAML==6.0.12.10
typing_extensions==4.6.3
tzdata==2023.3
time-machine==2.13.0
tornado==6.4
types-python-dateutil==2.8.19.20240106
typing_extensions==4.9.0
tzdata==2023.4
uc-micro-py==1.0.2
ulid==1.1
unicodecsv==0.14.1
urllib3==1.26.16
uWSGI==2.0.22
vine==5.0.0
wcwidth==0.2.6
universal-pathlib==0.1.4
urllib3==1.26.18
uWSGI==2.0.24
vine==5.1.0
wcwidth==0.2.13
WebOb==1.8.7
websocket-client==1.5.3
websocket-client==1.7.0
Werkzeug==2.2.3
wheel==0.40.0
wrapt==1.15.0
WTForms==3.0.1
yappi==1.4.0
yarl==1.9.2
zipp==3.15.0
wheel==0.41.2
wrapt==1.16.0
WTForms==3.1.2
xattr==0.10.1
yappi==1.6.0
yarl==1.9.4
zipp==3.17.0

View File

@ -17,7 +17,7 @@ allowlist_externals=
sh
deps=
-r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.8.txt
commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
@ -35,7 +35,7 @@ commands =
bash -c "rm -rf $HOME/airflow"
airflow version
airflow db init
airflow db upgrade
airflow db migrate
airflow info
airflow dags list
airflow dags list-import-errors
@ -57,14 +57,13 @@ commands =
bash -c "rm -rf $HOME/airflow"
airflow version
airflow db init
airflow db upgrade
airflow db migrate
airflow info
airflow dags list
airflow dags list-import-errors
airflow tasks test example_bash_operator runme_0
airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow tasks run example_bash_operator runme_0 2018-01-01
airflow dags state example_bash_operator 2018-01-01
pytest \
{posargs} \
--cov-branch \

View File

@ -1,28 +1,29 @@
arrow==1.2.3
certifi==2023.5.7
charset-normalizer==3.1.0
click==8.1.3
click-default-group==1.2.2
arrow==1.3.0
certifi==2023.11.17
charset-normalizer==3.3.2
click==8.1.7
click-default-group==1.2.4
Flask==2.2.5
idna==3.4
importlib-metadata==4.13.0
iso8601==2.0.0
idna==3.6
importlib-metadata==6.11.0
iso8601==2.1.0
itsdangerous==2.1.2
Jinja2==3.1.2
Jinja2==3.1.3
keystoneauth1==4.3.1
MarkupSafe==2.1.3
os-service-types==1.7.0
pbr==5.11.1
pip==23.1.2
pbr==6.0.0
pip==23.2.1
PTable==0.9.2
pylibyaml==0.1.0
python-dateutil==2.8.2
PyYAML==6.0
PyYAML==6.0.1
requests==2.31.0
setuptools==67.7.2
setuptools==68.2.2
six==1.16.0
stevedore==5.1.0
urllib3==1.26.16
types-python-dateutil==2.8.19.20240106
urllib3==1.26.18
Werkzeug==2.2.3
wheel==0.40.0
zipp==3.15.0
wheel==0.41.2
zipp==3.17.0

View File

@ -18,7 +18,7 @@ allowlist_externals=
sh
deps=
-r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.8.txt
commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"

View File

@ -133,6 +133,7 @@
- name: Deploy Memcached using Airship
shell: |
set -ex
./tools/deployment/airskiff/developer/100-deploy-osh.sh
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"