airship
/
shipyard
Code Issues Proposed changes

Airflow stable 2.6.2 

This PS updates python modules and code to match Airflow 2.6.2 as well
as deploys new Airflow:

- bionic py36 gates  were removed
- python code corrected to match new modules versions
- selection of python modules versions was performed based on
  airflow-2.6.2 constraints
- airskiff deploy pipeline was aligned with latest in treasuremap v1.9
- shipyard chart was corrected to match new airflow cli, configuration
  items and their default values
- added new celery configuration items and their values
- updated airflow runtime logging config
- disabled deprecation and future python warnings in airflow images
- added celery to the list of airflow providers
- adjusted airflow runtime scripts to match new cli
- shipyard SQL queries to airflow DB were adjusted to match new SQL
  schema of the db
- shipyard_airflow and shipyard_client unit tests were updated to match
  new DB structure and new cli
- airflow db sync job is using db upgrade command
- helm version uplifted to v3.12.2

Change-Id: Ife88e53ce0dd8dc77bf267de1f5e6b8361ca76fd
This commit is contained in:
Sergiy Markin 2023-07-01 06:42:55 +00:00
parent 3fcc6e0d65
commit 81066ae98f
48 changed files with 607 additions and 862 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -49,6 +49,7 @@ nosetests.xml
coverage.xml
*.cover
.hypothesis/
cover/
# Translations
*.mo
@ -126,4 +127,4 @@ airflow.cfg
airflow.db
latest
src/bin/shipyard_airflow/shipyard_airflow/config
src/bin/shipyard_airflow/shipyard_airflow/webserver_config.py
src/bin/shipyard_airflow/shipyard_airflow/webserver_config.py

71
.zuul.yaml
View File

@ -14,36 +14,30 @@
templates:
- docs-on-readthedocs
- openstack-python38-jobs
- openstack-python36-jobs
vars:
rtd_webhook_id: '38576'
rtd_project_name: 'airship-shipyard'
ensure_global_symlinks: true
check:
jobs:
- openstack-tox-pep8
- airship-shipyard-openstack-tox-pep8-focal
- airship-shipyard-openstack-tox-cover-focal
- airship-shipyard-chart-build-gate
- airship-shipyard-chart-build-latest-htk
- airship-shipyard-whitespace-lint-gate
- airship-shipyard-airskiff-deployment-bionic
- airship-shipyard-airskiff-deployment-focal
- airship-shipyard-image-gate-ubuntu_focal
- airship-shipyard-image-gate-ubuntu_bionic
gate:
jobs:
- openstack-tox-pep8
- airship-shipyard-openstack-tox-pep8-focal
- airship-shipyard-openstack-tox-cover-focal
- airship-shipyard-chart-build-gate
- airship-shipyard-whitespace-lint-gate
- airship-shipyard-airskiff-deployment-bionic
- airship-shipyard-airskiff-deployment-focal
- airship-shipyard-image-gate-ubuntu_focal
- airship-shipyard-image-gate-ubuntu_bionic
post:
jobs:
- airship-shipyard-docker-build-publish-ubuntu_focal
- airship-shipyard-docker-build-publish-ubuntu_bionic
- shipyard-upload-git-mirror
- nodeset:
@ -58,6 +52,12 @@
- name: primary
label: ubuntu-focal
- job:
name: airship-shipyard-openstack-tox-pep8-focal
parent: openstack-tox-pep8
description: Runs pep8 job on focal
nodeset: airship-shipyard-single-node-focal
- job:
name: airship-shipyard-openstack-tox-cover-focal
@ -116,19 +116,6 @@
dynamic:
patch_set: true
- job:
name: airship-shipyard-image-gate-ubuntu_bionic
description: |
Run shipyard-image build for ubuntu_bionic
parent: airship-shipyard-image-base
irrelevant-files: *irrelevant-files
vars:
publish: false
distro: ubuntu_bionic
tags:
dynamic:
patch_set: true
- job:
name: airship-shipyard-airskiff-deployment-focal
nodeset: airship-shipyard-single-node-focal
@ -145,34 +132,14 @@
- airship/treasuremap
vars:
CLONE_SHIPYARD: false
HTK_COMMIT: master
HTK_COMMIT: 97ce6d7d8e9a090c748800d69a57bbd9af698b60
SHIPYARD_IMAGE_DISTRO: ubuntu_focal
irrelevant-files: *irrelevant-files
- job:
name: airship-shipyard-airskiff-deployment-bionic
nodeset: airship-shipyard-single-node-focal
description: |
Deploy Memcached using Airskiff and submitted Armada changes.
timeout: 9600
voting: true
pre-run:
- tools/gate/playbooks/airskiff-reduce-site.yaml
- tools/gate/playbooks/git-config.yaml
run: tools/gate/playbooks/airskiff-deploy.yaml
post-run: tools/gate/playbooks/debug-report.yaml
required-projects:
- airship/treasuremap
vars:
CLONE_SHIPYARD: false
HTK_COMMIT: master
SHIPYARD_IMAGE_DISTRO: ubuntu_bionic
irrelevant-files: *irrelevant-files
- job:
name: airship-shipyard-docker-build-publish-ubuntu_focal
timeout: 1800
timeout: 3600
run: tools/gate/playbooks/run-image.yaml
nodeset: airship-shipyard-single-node-focal
irrelevant-files: *irrelevant-files
@ -187,24 +154,8 @@
commit: true
static:
- latest
- airflow_2.6.2
- job:
name: airship-shipyard-docker-build-publish-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/run-image.yaml
nodeset: airship-shipyard-single-node-focal
irrelevant-files: *irrelevant-files
secrets:
- airship_shipyard_quay_creds
vars:
publish: true
distro: ubuntu_bionic
tags:
dynamic:
branch: true
commit: true
static:
- latest
- secret:

2
charts/shipyard/Chart.yaml
View File

@ -16,7 +16,7 @@
apiVersion: v1
description: A Helm chart for Shipyard and Airflow
name: shipyard
version: 0.1.2
version: 0.2.0
keywords:
- shipyard
- airflow

2
charts/shipyard/templates/bin/_airflow-db-sync.sh.tpl
View File

@ -19,4 +19,4 @@ limitations under the License.
set -ex
airflow initdb
airflow db upgrade

2
charts/shipyard/templates/deployment-airflow-scheduler.yaml
View File

@ -89,7 +89,7 @@ spec:
subPath: shipyard.conf
readOnly: true
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
mountPath: {{ .Values.conf.airflow.logging.base_log_folder }}
{{ if $mounts_airflow_scheduler.volumeMounts }}{{ toYaml $mounts_airflow_scheduler.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: airflow-etc

2
charts/shipyard/templates/deployment-shipyard.yaml
View File

@ -142,7 +142,7 @@ spec:
subPath: shipyard.conf
readOnly: true
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
mountPath: {{ .Values.conf.airflow.logging.base_log_folder }}
{{ if $mounts_shipyard.volumeMounts }}{{ toYaml $mounts_shipyard.volumeMounts | indent 12 }}{{ end }}
volumes:
{{ if .Values.conf.shipyard.base.profiler }}

12
charts/shipyard/templates/statefulset-airflow-worker.yaml
View File

@ -120,10 +120,10 @@ spec:
- chown
- -R
- "airflow:airflow"
- {{ .Values.conf.airflow.core.base_log_folder }}
- {{ .Values.conf.airflow.logging.base_log_folder }}
volumeMounts:
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
mountPath: {{ .Values.conf.airflow.logging.base_log_folder }}
containers:
- name: airflow-scheduler
image: {{ .Values.images.tags.airflow }}
@ -152,7 +152,7 @@ spec:
subPath: shipyard.conf
readOnly: true
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
mountPath: {{ .Values.conf.airflow.logging.base_log_folder }}
- name: airflow-worker
image: {{ .Values.images.tags.airflow }}
imagePullPolicy: {{ .Values.images.pull_policy }}
@ -184,7 +184,7 @@ spec:
subPath: shipyard.conf
readOnly: true
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
mountPath: {{ .Values.conf.airflow.logging.base_log_folder }}
- name: airflow-logrotate
image: {{ .Values.images.tags.airflow }}
imagePullPolicy: {{ .Values.images.pull_policy }}
@ -197,12 +197,12 @@ spec:
- name: DAYS_BEFORE_LOG_DELETION
value: {{ .Values.logrotate.days_before_deletion | quote }}
- name: LOGROTATE_PATH
value: {{ .Values.conf.airflow.core.base_log_folder }}
value: {{ .Values.conf.airflow.logging.base_log_folder }}
- name: LOGROTATE_ENABLED
value: "enabled"
volumeMounts:
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
mountPath: {{ .Values.conf.airflow.logging.base_log_folder }}
volumes:
- name: pod-tmp
emptyDir: {}

85
charts/shipyard/values.yaml
View File

@ -471,34 +471,13 @@ conf:
# Envrionment variable AIRFLOW_HOME is used instead.
#airflow_home: /usr/local/airflow
dags_folder: /usr/local/airflow/dags
base_log_folder: /usr/local/airflow/logs
remote_logging: "False"
remote_log_conn_id: ""
remote_base_log_folder: ""
encrypt_s3_logs: "False"
logging_level: "INFO"
fab_logging_level: "WARN"
# See image-bundled log_config.py.
# Adds console logging of task/step logs.
logging_config_class: log_config.LOGGING_CONFIG
# NOTE: Airflow 1.10 introduces extra newline characters between log
# records. Version 1.10.1 should resolve this issue
# https://issues.apache.org/jira/browse/AIRFLOW-1917
#
# NOTE: The log format ends up repeated for each log record that we log
# in our custom operators, once for the logging_mixin class of
# Airflow itself, and once again for the message we want to log.
# E.g.:
# 2018-09-21 19:38:48,950 INFO logging_mixin(95) write - 2018-09-21 19:38:48,950 INFO deployment_configuration_operator(135) get_doc - Deckhand Client acquired
#
# NOTE: Updated from default to match Shipyard logging as much as
# possible without more aggressive techniques
#
log_format: "%%(asctime)s %%(levelname)-8s %%(filename)s:%%(lineno)3d:%%(funcName)s %%(module)s %%(message)s"
simple_log_format: "%%(asctime)s %%(levelname)s - %%(message)s"
log_filename_template: "{{ ti.dag_id }}/{{ ti.task_id }}/{{ execution_date.strftime('%%Y-%%m-%%dT%%H:%%M:%%S') }}/{{ try_number }}.log"
log_processor_filename_template: "{{ filename }}.log"
dag_processor_manager_log_location: /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log
sensitive_var_conn_names:
- sql_alchemy_conn
- broker_url
- result_backend
- fernet_key
lazy_discover_providers: "False"
lazy_load_plugins: "False"
hostname_callable: "socket:getfqdn"
default_timezone: "utc"
executor: "CeleryExecutor"
@ -539,9 +518,40 @@ conf:
endpoint_url: http://localhost/
api:
auth_backend: airflow.api.auth.backend.default
enable_experimental_api: "True"
lineage:
# Shipyard is not using this
backend: ""
logging:
# See image-bundled log_config.py.
# Adds console logging of task/step logs.
# logging_config_class: log_config.LOGGING_CONFIG
logging_config_class: new_log_config.LOGGING_CONFIG
# NOTE: Airflow 1.10 introduces extra newline characters between log
# records. Version 1.10.1 should resolve this issue
# https://issues.apache.org/jira/browse/AIRFLOW-1917
#
# NOTE: The log format ends up repeated for each log record that we log
# in our custom operators, once for the logging_mixin class of
# Airflow itself, and once again for the message we want to log.
# E.g.:
# 2018-09-21 19:38:48,950 INFO logging_mixin(95) write - 2018-09-21 19:38:48,950 INFO deployment_configuration_operator(135) get_doc - Deckhand Client acquired
#
# NOTE: Updated from default to match Shipyard logging as much as
# possible without more aggressive techniques
#
log_format: "%%(asctime)s %%(levelname)-8s %%(filename)s:%%(lineno)3d:%%(funcName)s %%(module)s %%(message)s"
log_filename_template: "{{ ti.dag_id }}/{{ ti.task_id }}/{{ execution_date.strftime('%%Y-%%m-%%dT%%H:%%M:%%S') }}/{{ try_number }}.log"
log_processor_filename_template: "{{ filename }}.log"
dag_processor_manager_log_location: /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log
logging_level: "INFO"
fab_logging_level: "WARNING"
celery_logging_level: "INFO"
base_log_folder: /usr/local/airflow/logs
remote_logging: "False"
remote_log_conn_id: ""
remote_base_log_folder: ""
encrypt_s3_logs: "False"
atlas:
# Shipyard is not using this
sasl_enabled: "False"
@ -617,6 +627,11 @@ conf:
flower_url_prefix: ""
flower_port: 5555
flower_basic_auth: ""
operation_timeout: 30.0
broker_connection_retry_on_startup: "True"
broker_connection_retry: "True"
broker_connection_max_retries: 0
broker_connection_timeout: 30.0
default_queue: "default"
# How many processes CeleryExecutor uses to sync task state.
# 0 means to use max(1, number of cores - 1) processes.
@ -645,7 +660,6 @@ conf:
# scheduler section in the docs for more information). This defines
# how often the scheduler should run (in seconds).
scheduler_heartbeat_sec: 10
run_duration: -1
# Check for pending dag runs no more than every 10 seconds
min_file_process_interval: 10
dag_dir_list_interval: 300
@ -655,16 +669,19 @@ conf:
scheduler_zombie_task_threshold: 300
catchup_by_default: "True"
max_tis_per_query: 512
statsd_on: "False"
statsd_host: "localhost"
statsd_port: 8125
statsd_prefix: "airflow"
# Shipyard's use of Airflow is low volume. 1 Thread is probably enough.
max_threads: 1
# deprecated in 1.10.14. Replaced with "parsing_processes"
# max_threads: 1
parsing_processes: 1
authenticate: "False"
# Turn off scheduler use of cron intervals by setting this to False.
# DAGs submitted manually in the web UI or with trigger_dag will still run.
use_job_schedule: "False"
metrics:
statsd_on: "False"
statsd_host: "localhost"
statsd_port: 8125
statsd_prefix: "airflow"
ldap:
# Shipyard is not using this
uri: ""

14
images/airflow/Dockerfile.ubuntu_focal
View File

@ -32,6 +32,7 @@ ENV WEB_PORT 8080
ENV FLOWER_PORT 5555
ENV WORKER_PORT 8793
ENV SLUGIFY_USES_TEXT_UNIDECODE yes
ENV PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
# Expose port for applications
EXPOSE $WEB_PORT
@ -43,12 +44,12 @@ ARG AIRFLOW_HOME=/usr/local/airflow
# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
# version of celery incompatibile with the version of kombu needed by other
# Airship components
ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5"
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.6.2"
ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin
# Kubectl version
ARG KUBECTL_VERSION=1.26.3
ARG KUBECTL_VERSION=1.27.1
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
@ -124,8 +125,8 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
# Install Airflow directly to allow overriding source
# COPY images/airflow/requirements.txt /tmp/
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/requirements.txt
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir
# && pip3 install $AIRFLOW_SRC --no-cache-dir
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
&& pip3 install $AIRFLOW_SRC --no-cache-dir
# && (pip3 uninstall -y snakebite || true) \
# && (pip3 uninstall -y psycopg2 || true) \
# && (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true)
@ -135,6 +136,7 @@ COPY images/airflow/script/*.sh ${AIRFLOW_HOME}/
# Copy configuration (e.g. logging config for Airflow):
COPY images/airflow/config/*.py ${AIRFLOW_HOME}/config/
COPY images/airflow/webserver_config.py ${AIRFLOW_HOME}/
# Change permissions
RUN chown -R airflow: ${AIRFLOW_HOME}
@ -167,7 +169,11 @@ RUN cd /tmp/shipyard \
#
# Copy the plugins and dags that will be used by this Airflow image:
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins ${AIRFLOW_HOME}/plugins/
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/plugins \
/usr/local/lib/python3.8/dist-packages/airflow/plugins/
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags ${AIRFLOW_HOME}/dags/
COPY ${ctx_base}/shipyard_airflow/shipyard_airflow/dags \
/usr/local/lib/python3.8/dist-packages/airflow/dags/
# Set work directory
USER airflow

4
images/airflow/config/new_log_config.py Normal file
View File

@ -0,0 +1,4 @@
from copy import deepcopy
from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
LOGGING_CONFIG = deepcopy(DEFAULT_LOGGING_CONFIG)

19
images/airflow/script/airflow_start_service.sh
View File

@ -19,15 +19,15 @@ python3_path=$(which python3)
airflow_path=$(which airflow)
# Initialize Airflow DB
if [[ $cmd == 'initdb' ]]; then
${python3_path} ${airflow_path} initdb
if [[ $cmd == 'db init' ]]; then
${python3_path} ${airflow_path} db init
# Start the services based on argument from Airflow Helm Chart
elif [[ $cmd == 'webserver' ]]; then
${python3_path} ${airflow_path} webserver
elif [[ $cmd == 'flower' ]]; then
${python3_path} ${airflow_path} flower
${python3_path} ${airflow_path} celery flower --pid=/tmp/airflow-flower.pid
elif [[ $cmd == 'worker' ]]; then
${python3_path} ${airflow_path} worker
${python3_path} ${airflow_path} celery worker --pid=/tmp/airflow-worker.pid
# If command contains the word 'scheduler'
elif [[ $cmd == *scheduler* ]]; then
while true; do
@ -39,11 +39,14 @@ elif [[ $cmd == *scheduler* ]]; then
${python3_path} ${airflow_path} scheduler $2 $3
done
elif [[ $cmd == 'quicktest' ]]; then
${python3_path} ${airflow_path} initdb
${python3_path} ${airflow_path} db init
${python3_path} ${airflow_path} db upgrade
${python3_path} ${airflow_path} dags list
${python3_path} ${airflow_path} webserver -p 8080 &
airflow run example_bash_operator runme_0 2018-01-01
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow dag_state example_bash_operator 2018-01-01
${python3_path} ${airflow_path} tasks test example_bash_operator runme_0
${python3_path} ${airflow_path} dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
${python3_path} ${airflow_path} tasks run example_bash_operator runme_0 2018-01-01
${python3_path} ${airflow_path} dags state example_bash_operator 2018-01-01
else
echo "Invalid Command!"
exit 1

6
images/airflow/script/upgrade_airflow_worker.sh
View File

@ -64,12 +64,12 @@ do
# Get current state of dag using Airflow CLI
# Use grep to remove logging messages that can pollute the status response
check_dag_state=`airflow dag_state ${dag_id} ${dag_execution_date} | grep -vE "DEBUG|INFO|WARN|ERROR"`
check_dag_state=$(airflow dags state ${dag_id} ${dag_execution_date} | grep -vE "DEBUG|INFO|WARN|ERROR")
echo -e ${check_dag_state} >> /usr/local/airflow/upgrade_airflow_worker.log
# We will need to extract the last word in the 'check_dag_state'
# string variable as that will contain the status of the dag run
dag_state=`echo ${check_dag_state} | awk '{print $NF}'`
dag_state=$(echo ${check_dag_state} | awk '{print $NF}')
echo -e ${dag_state} >> /usr/local/airflow/upgrade_airflow_worker.log
if [[ $dag_state == "success" ]]; then
@ -78,7 +78,7 @@ do
echo -e "Proceeding to upgrade Airflow Worker..." >> /usr/local/airflow/upgrade_airflow_worker.log
echo -e "Deleting Airflow Worker Pods..." >> /usr/local/airflow/upgrade_airflow_worker.log
for i in `kubectl get pods -n ucp | grep -i airflow-worker | awk '{print $1}'`; do
for i in $(kubectl get pods -n ucp | grep -i airflow-worker | awk '{print $1}'); do
# Delete Airflow Worker pod so that they will respawn with the new
# configurations and/or images
kubectl delete pod $i -n ucp

10
images/airflow/webserver_config.py Normal file
View File

@ -0,0 +1,10 @@
"""Default configuration for the Airflow webserver."""
from __future__ import annotations
import os
from airflow.www.fab_security.manager import AUTH_DB
basedir = os.path.abspath(os.path.dirname(__file__))
WTF_CSRF_ENABLED = True
WTF_CSRF_TIME_LIMIT = None
AUTH_TYPE = AUTH_DB
AUTH_ROLE_ADMIN = 'Admin'
AUTH_ROLE_PUBLIC = 'Admin'

7
images/shipyard/Dockerfile.ubuntu_focal
View File

@ -26,6 +26,9 @@ ENV container docker
ENV PORT 9000
ENV LC_ALL C.UTF-8
ENV LANG C.UTF-8
ENV PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
# Setting the version explicitly for PBR
ENV PBR_VERSION 0.1a1
@ -112,10 +115,10 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
&& python3 -m pip install -U pip \
&& pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \
&& cd /home/shipyard/shipyard_client \
&& python3 setup.py install \
&& pip3 install . --use-pep517 --no-cache-dir \
&& pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \
&& cd /home/shipyard/shipyard \
&& python3 setup.py install \
&& pip3 install . --use-pep517 --no-cache-dir \
&& apt-get purge -y --auto-remove $buildDeps \
&& apt-get autoremove -yqq --purge \
&& apt-get clean \

2
src/bin/shipyard_airflow/entrypoint.sh
View File

@ -50,7 +50,7 @@ if [ "$1" = 'server' ]; then
--http :${PORT} \
--paste config:/etc/shipyard/api-paste.ini \
--enable-threads \
-L \
--disable-logging \
--pyargv "--config-file /etc/shipyard/shipyard.conf" \
--threads ${SHIPYARD_API_THREADS} \
--workers ${SHIPYARD_API_WORKERS} \

85
src/bin/shipyard_airflow/requirements-direct.txt
View File

@ -1,40 +1,55 @@
alembic==1.0.1
apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5
arrow<=0.17.0
celery==4.3
Flask==1.1.0
Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@airflow_v1.13.2
pytz==2018.5
coverage==5.3
alembic
apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.6.2
arrow
celery
Flask
Flask-AppBuilder
pytz
falcon
jsonschema<=3.2.0
keystoneauth1<=5.1.1
kombu
kubernetes==26.1.0
marshmallow-sqlalchemy==0.18.0
jsonschema
networkx
oslo.config<=8.7.1
oslo.policy<=3.10.1
oslo.utils<=4.12.3
psycopg2-binary==2.8.4
pyarrow==6.0.1
pylibyaml==0.1.0
PyYAML<=5.4.1
regex==2020.11.13
requests==2.23.0
responses==0.12.1
setuptools<=45.2.0
SQLAlchemy==1.3.15
tabulate==0.8.03
ulid==1.1
uWSGI==2.0.21
Werkzeug<=0.16.1
WTForms<=2.3.3
keystoneauth1==4.3.1
keystonemiddleware==9.3.0
kombu
kubernetes
marshmallow-sqlalchemy
mock
oslo.cache==2.10.0
oslo.concurrency==4.2.0
oslo.config==8.6.0
oslo.context==4.0.0
oslo.db==11.2.0
oslo.i18n==6.0.0
oslo.log==4.8.0
oslo.messaging==12.13.1
oslo.metrics==0.4.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==4.1.0
oslo.service==2.8.0
oslo.upgradecheck==2.1.1
oslo.utils==4.12.3
oslo.versionedobjects==2.4.0
psycopg2-binary
pyarrow
pylibyaml
python3-memcached
PyYAML
regex
requests
responses
setuptools
SQLAlchemy
tabulate
ulid
uWSGI
Werkzeug
WTForms
itsdangerous==2.0.1
itsdangerous
git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5#egg=deckhand
git+https://opendev.org/airship/drydock.git@73d22a488f7f5b7217e86e86da9f38b5b9085009#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@812546c87595b1d8a1bc0da931d644473b03b31f#egg=armada
git+https://opendev.org/airship/promenade.git@2f2455f0d159287565a279a99c0bbeb235e81ffb#egg=promenade
git+https://opendev.org/airship/deckhand.git@37ba567da5bf7f146f54a468478a025c70793574#egg=deckhand
git+https://opendev.org/airship/drydock.git@f99abfa4337f8cbb591513aac404b11208d4187c#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@6595dd83ea65324196c89cf6fb83f168818822de#egg=armada
git+https://opendev.org/airship/promenade.git@69a74590e76e810916f7780fc525c63ec58c7dc1#egg=promenade

241
src/bin/shipyard_airflow/requirements-frozen-bionic.txt
View File

@ -1,241 +0,0 @@
alabaster==0.7.13
alembic==1.0.1
amqp==2.6.1
apache-airflow==1.10.5
apispec==1.3.3
argcomplete==3.0.8
Armada @ git+https://opendev.org/airship/armada.git@812546c87595b1d8a1bc0da931d644473b03b31f
arrow==0.17.0
attrs==22.2.0
autopage==0.5.1
Babel==2.11.0
bcrypt==4.0.1
Beaker==1.12.0
billiard==3.6.4.0
cached-property==1.5.2
cachetools==4.2.4
celery==4.3.0
certifi==2023.5.7
cffi==1.15.1
chardet==3.0.4
charset-normalizer==3.0.1
click==7.1.2
cliff==3.10.1
cmd2==2.4.3
colorama==0.4.5
colorlog==4.0.2
configparser==3.5.3
coverage==5.3
croniter==0.3.37
cryptography==3.4.8
debtcollector==2.5.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@06b1631d8a31a1e7147e8751025d0ef2ff4b31f5
decorator==4.4.2
deepdiff==5.7.0
defusedxml==0.7.1
dill==0.2.9
distlib==0.3.6
dnspython==2.2.1
docutils==0.17
dogpile.cache==1.2.1
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@73d22a488f7f5b7217e86e86da9f38b5b9085009#subdirectory=python
dulwich==0.20.50
dumb-init==1.2.5.post1
eventlet==0.33.3
extras==1.0.0
falcon==3.1.1
fasteners==0.18
filelock==3.4.1
fixtures==3.0.0
flake8==3.8.4
Flask==1.1.0
Flask-Admin==1.5.3
Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@b71789d85b632935eca79c0b53fb77f20bf17fe6
Flask-Babel==0.12.2
Flask-Caching==1.3.3
Flask-JWT-Extended==3.25.1
Flask-Login==0.4.1
Flask-OpenID==1.3.0
Flask-SQLAlchemy==2.5.1
flask-swagger==0.2.13
Flask-WTF==0.14.3
flower==0.9.7
funcsigs==1.0.0
future==0.16.0
futurist==2.4.1
gitdb==4.0.9
GitPython==3.1.18
google-auth==2.19.0
greenlet==2.0.2
gunicorn==19.10.0
hacking==4.1.0
hmsclient==0.1.1
html5lib==0.9999999
httpexceptor==1.4.0
humanize==3.14.0
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
importlib-resources==5.4.0
iso8601==1.1.0
itsdangerous==2.0.1
JayDeBeApi==1.2.3
Jinja2==2.10.3
JPype1==1.3.0
json-merge-patch==0.2
jsonpath-ng==1.5.3
jsonpath-rw==1.4.0
jsonpath-rw-ext==1.2.2
jsonpickle==1.4.1
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==9.5.0
kombu==4.6.3
kubernetes==26.1.0
lazy-object-proxy==1.7.1
lockfile==0.12.2
Mako==1.1.6
Markdown==2.6.11
MarkupSafe==2.0.1
marshmallow==2.19.5
marshmallow-enum==1.5.1
marshmallow-sqlalchemy==0.18.0
mccabe==0.6.1
mock==5.0.2
msgpack==1.0.5
natsort==8.2.0
netaddr==0.8.0
netifaces==0.11.0
networkx==2.5.1
nose==1.3.7
numpy==1.19.5
oauthlib==3.2.2
ordered-set==4.0.2
ordereddict==1.1
os-service-types==1.7.0
oslo.cache==2.10.1
oslo.concurrency==4.5.1
oslo.config==8.7.1
oslo.context==4.1.0
oslo.db==10.0.0
oslo.i18n==5.1.0
oslo.log==4.6.0
oslo.messaging==10.3.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==4.2.0
oslo.service==2.8.0
oslo.utils==4.12.3
packaging==21.3
pandas==0.25.3
Paste==3.5.0
PasteDeploy==2.1.1
PasteScript==3.3.0
pbr==5.5.1
pendulum==1.4.4
pip==21.3.1
platformdirs==2.4.0
ply==3.11
prettytable==2.5.0
prison==0.1.0
promenade @ git+https://opendev.org/airship/promenade.git@2f2455f0d159287565a279a99c0bbeb235e81ffb
prometheus-client==0.8.0
protobuf==3.19.6
psutil==5.9.5
psycopg2-binary==2.8.4
py==1.11.0
pyarrow==6.0.1
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycadf==3.1.1
pycodestyle==2.6.0
pycparser==2.21
pyflakes==2.2.0
Pygments==2.14.0
PyHive==0.6.5
PyJWT==1.7.1
pylibyaml==0.1.0
pymongo==4.1.1
pyparsing==2.4.7
pyperclip==1.8.2
pyproject-api==0.1.0
pyrsistent==0.18.0
python-barbicanclient==5.2.0
python-daemon==2.1.2
python-dateutil==2.8.1
python-editor==1.0.4
python-keystoneclient==4.5.0
python-memcached==1.59
python-mimeparse==1.6.0
python-subunit==1.4.0
python3-openid==3.2.0
pytz==2018.5
pytzdata==2020.1
PyYAML==5.4.1
regex==2020.11.13
reno==4.0.0
repoze.lru==0.7
requests==2.23.0
requests-oauthlib==1.3.1
resolver==0.2.1
responses==0.12.1
retry==0.9.2
rfc3986==1.5.0
Routes==2.5.1
rsa==4.9
selector==0.10.1
setproctitle==1.2.3
setuptools==45.2.0
simplejson==3.19.1
six==1.16.0
smmap==5.0.0
snakebite==2.11.0
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
SQLAlchemy==1.3.15
sqlalchemy-migrate==0.13.0
sqlparse==0.4.4
statsd==4.0.1
stevedore==3.5.2
tabulate==0.8.3
Tempita==0.5.2
tenacity==4.12.0
termcolor==1.1.0
testrepository==0.0.20
testresources==2.0.1
testscenarios==0.5.0
testtools==2.5.0
text-unidecode==1.2
thrift==0.16.0
tiddlyweb==2.4.3
tomli==1.2.3
tomlkit==0.11.6
tornado==5.1.1
typing-extensions==3.7.4.3
tzlocal==1.5.1
ulid==1.1
unicodecsv==0.14.1
urllib3==1.25.11
uWSGI==2.0.21
vine==1.3.0
virtualenv==20.16.2
wcwidth==0.2.6
WebOb==1.8.7
websocket-client==1.3.1
Werkzeug==0.16.1
wheel==0.37.1
wrapt==1.15.0
wsgi-intercept==1.11.0
WTForms==2.3.3
xmltodict==0.13.0
yappi==1.4.0
yq==3.2.1
zipp==3.6.0
zope.deprecation==4.4.0

333
src/bin/shipyard_airflow/requirements-frozen.txt
View File

@ -1,240 +1,269 @@
alabaster==0.7.13
alembic==1.0.1
amqp==2.6.1
apache-airflow==1.10.5
apispec==1.3.3
argcomplete==3.0.8
Armada @ git+https://opendev.org/airship/armada.git@4be7fa44a743fbcdf792e2bdca3af6c95b540f97
arrow==0.17.0
aiohttp==3.8.4
aiosignal==1.3.1
alembic==1.11.1
amqp==5.1.1
anyio==3.7.0
apache-airflow==2.6.2
apache-airflow-providers-apache-hdfs==4.0.0
apache-airflow-providers-apache-hive==6.1.0
apache-airflow-providers-celery==3.2.0
apache-airflow-providers-common-sql==1.5.1
apache-airflow-providers-ftp==3.4.1
apache-airflow-providers-http==4.4.1
apache-airflow-providers-imap==3.2.1
apache-airflow-providers-jdbc==3.4.0
apache-airflow-providers-postgres==5.5.0
apache-airflow-providers-sqlite==3.4.1
apispec==5.2.2
argcomplete==3.1.1
Armada @ git+https://opendev.org/airship/armada.git@6595dd83ea65324196c89cf6fb83f168818822de
arrow==1.2.3
asgiref==3.7.2
async-timeout==4.0.2
attrs==23.1.0
autopage==0.5.1
Babel==2.12.1
backports.zoneinfo==0.2.1
barbican==14.0.2
bcrypt==4.0.1
Beaker==1.12.0
billiard==3.6.4.0
cached-property==1.5.2
cachetools==5.3.0
celery==4.3.0
Beaker==1.12.1
billiard==4.1.0
blinker==1.6.2
cachelib==0.9.0
cachetools==5.3.1
castellan==4.1.0
cattrs==23.1.2
celery==5.3.0
certifi==2023.5.7
cffi==1.15.1
chardet==3.0.4
charset-normalizer==3.1.0
click==7.1.2
cliff==3.10.1
click==8.1.3
click-didyoumean==0.3.0
click-plugins==1.1.1
click-repl==0.2.0
clickclick==20.10.2
cliff==4.3.0
cmd2==2.4.3
colorama==0.4.6
colorlog==4.0.2
configparser==3.5.3
coverage==5.3
croniter==0.3.37
cryptography==3.4.8
colorlog==4.8.0
ConfigUpdater==3.1.1
connexion==2.14.2
cron-descriptor==1.4.0
croniter==1.3.15
cryptography==40.0.2
debtcollector==2.5.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@3a06b1b6040865483a09514e2bf7dc5fae24560e
Deckhand @ git+https://opendev.org/airship/deckhand.git@37ba567da5bf7f146f54a468478a025c70793574
decorator==5.1.1
deepdiff==5.8.1
defusedxml==0.7.1
dill==0.2.9
distlib==0.3.6
deepdiff==6.3.1
Deprecated==1.2.14
dill==0.3.1.1
dnspython==2.3.0
docutils==0.17
dogpile.cache==1.2.0
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@375abedb8aa3a413ca70a47aef467f39d65e1aee#subdirectory=python
dulwich==0.21.5
dumb-init==1.2.5.post1
docopt==0.6.2
docutils==0.20.1
dogpile.cache==1.2.2
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@f99abfa4337f8cbb591513aac404b11208d4187c#subdirectory=python
email-validator==1.3.1
eventlet==0.33.3
extras==1.0.0
exceptiongroup==1.1.1
falcon==3.1.1
fastavro==1.7.4
fasteners==0.18
filelock==3.12.0
fixtures==3.0.0
flake8==3.8.4
Flask==1.1.0
Flask-Admin==1.5.3
Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@b71789d85b632935eca79c0b53fb77f20bf17fe6
Flask-Babel==0.12.2
Flask-Caching==1.3.3
Flask-JWT-Extended==3.25.1
Flask-Login==0.4.1
Flask-OpenID==1.3.0
fixtures==4.1.0
Flask==2.2.5
Flask-AppBuilder==4.3.1
Flask-Babel==2.0.0
Flask-Caching==2.0.2
Flask-JWT-Extended==4.5.2
Flask-Limiter==3.3.1
Flask-Login==0.6.2
Flask-Session==0.5.0
Flask-SQLAlchemy==2.5.1
flask-swagger==0.2.13
Flask-WTF==0.14.3
flower==0.9.7
funcsigs==1.0.0
future==0.16.0
Flask-WTF==1.1.1
flower==1.2.0
frozenlist==1.3.3
future==0.18.3
futurist==2.4.1
gitdb==4.0.10
GitPython==3.1.31
google-auth==2.18.0
google-auth==2.20.0
graphviz==0.20.1
greenlet==2.0.2
gunicorn==19.10.0
hacking==4.1.0
gssapi==1.8.2
gunicorn==20.1.0
h11==0.14.0
hdfs==2.7.0
hmsclient==0.1.1
html5lib==0.9999999
httpcore==0.16.3
httpexceptor==1.4.0
httpx==0.23.3
humanize==4.6.0
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
iso8601==1.1.0
itsdangerous==2.0.1
idna==3.4
importlib-metadata==4.13.0
importlib-resources==5.12.0
inflection==0.5.1
iso8601==2.0.0
itsdangerous==2.1.2
JayDeBeApi==1.2.3
Jinja2==2.10.3
Jinja2==3.1.2
JPype1==1.4.1
json-merge-patch==0.2
jsonpath-ng==1.5.3
jsonpath-rw==1.4.0
jsonpath-rw-ext==1.2.2
jsonpickle==1.4.1
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==10.2.0
kombu==4.6.3
kubernetes==26.1.0
jsonpickle==3.0.1
jsonschema==4.17.3
keystoneauth1==4.3.1
keystonemiddleware==9.3.0
kombu==5.3.0
krb5==0.5.0
kubernetes==23.6.0
lazy-object-proxy==1.9.0
ldap3==2.9.1
limits==3.5.0
linkify-it-py==2.0.2
lockfile==0.12.2
logutils==0.3.5
Mako==1.2.4
Markdown==2.6.11
MarkupSafe==2.0.1
marshmallow==2.19.5
Markdown==3.4.3
markdown-it-py==3.0.0
MarkupSafe==2.1.3
marshmallow==3.19.0
marshmallow-enum==1.5.1
marshmallow-sqlalchemy==0.18.0
mccabe==0.6.1
mock==5.0.2
marshmallow-oneofschema==3.0.1
marshmallow-sqlalchemy==0.26.1
mdit-py-plugins==0.4.0
mdurl==0.1.2
mock==5.1.0
msgpack==1.0.5
natsort==8.3.1
multidict==6.0.4
netaddr==0.8.0
netifaces==0.11.0
networkx==3.1
nose==1.3.7
numpy==1.24.3
oauthlib==3.2.2
ordered-set==4.1.0
ordereddict==1.1
os-service-types==1.7.0
oslo.cache==2.10.1
oslo.concurrency==5.1.1
oslo.config==8.7.1
oslo.context==4.1.0
oslo.db==10.0.0
oslo.cache==2.10.0
oslo.concurrency==4.2.0
oslo.config==8.6.0
oslo.context==4.0.0
oslo.db==11.2.0
oslo.i18n==6.0.0
oslo.log==4.6.0
oslo.messaging==10.3.0
oslo.log==4.8.0
oslo.messaging==12.13.1
oslo.metrics==0.4.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==4.2.0
oslo.service==3.1.1
oslo.serialization==4.1.0
oslo.service==2.8.0
oslo.upgradecheck==2.1.1
oslo.utils==4.12.3
oslo.versionedobjects==2.4.0
packaging==21.3
pandas==0.25.3
Paste==3.5.0
pandas==1.5.3
Paste==3.5.3
PasteDeploy==3.0.1
PasteScript==3.3.0
pbr==5.5.1
pendulum==1.4.4
pathspec==0.9.0
pbr==5.11.1
pecan==1.5.1
pendulum==2.1.2
pip==23.1.2
platformdirs==3.5.1
pkgutil_resolve_name==1.3.10
pluggy==1.0.0
ply==3.11
prettytable==3.7.0
prison==0.1.0
promenade @ git+https://opendev.org/airship/promenade.git@45bcba068eb42fe6ba54d494b12122600dcb2c6c
prometheus-client==0.8.0
protobuf==4.23.0
prettytable==3.8.0
prison==0.2.1
promenade @ git+https://opendev.org/airship/promenade.git@69a74590e76e810916f7780fc525c63ec58c7dc1
prometheus-client==0.17.0
prompt-toolkit==3.0.38
psutil==5.9.5
psycopg2-binary==2.8.4
psycopg2-binary==2.9.6
pure-sasl==0.6.2
py==1.11.0
pyarrow==6.0.1
pyasn1==0.5.0
pyasn1-modules==0.3.0
pyarrow==11.0.0
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycadf==3.1.1
pycodestyle==2.6.0
pycparser==2.21
pyflakes==2.2.0
Pygments==2.14.0
pydantic==1.10.9
Pygments==2.15.1
PyHive==0.6.5
PyJWT==1.7.1
PyJWT==2.7.0
pylibyaml==0.1.0
pymongo==4.3.3
pyparsing==2.4.7
pyOpenSSL==23.2.0
pyparsing==3.0.9
pyperclip==1.8.2
pyproject_api==1.5.0
pyrsistent==0.19.3
python-barbicanclient==5.2.0
python-daemon==2.1.2
python-dateutil==2.8.1
python-editor==1.0.4
pyspnego==0.9.1
python-barbicanclient==5.4.0
python-daemon==3.0.1
python-dateutil==2.8.2
python-keystoneclient==5.1.0
python-memcached==1.59
python-mimeparse==1.6.0
python-subunit==1.4.0
python3-openid==3.2.0
pytz==2018.5
python-nvd3==0.15.0
python-slugify==8.0.1
python3-memcached==1.51
pytz==2023.3
pytzdata==2020.1
PyYAML==5.4.1
regex==2020.11.13
reno==4.0.0
PyYAML==6.0
regex==2023.6.3
repoze.lru==0.7
requests==2.23.0
requests==2.31.0
requests-kerberos==0.14.0
requests-oauthlib==1.3.1
requests-toolbelt==1.0.0
resolver==0.2.1
responses==0.12.1
responses==0.23.1
retry==0.9.2
rfc3986==2.0.0
rfc3339-validator==0.1.4
rfc3986==1.5.0
rich==13.4.2
rich_argparse==1.1.1
Routes==2.5.1
rsa==4.9
sasl==0.3.1
selector==0.10.1
setproctitle==1.3.2
setuptools==45.2.0
setuptools==67.7.2
simplejson==3.19.1
six==1.16.0
smmap==5.0.0
snakebite==2.11.0
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.4
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.1
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
SQLAlchemy==1.3.15
sniffio==1.3.0
SQLAlchemy==1.4.48
SQLAlchemy-JSONField==1.0.1.post0
sqlalchemy-migrate==0.13.0
SQLAlchemy-Utils==0.41.1
sqlparse==0.4.4
statsd==4.0.1
stevedore==5.1.0
tabulate==0.8.3
tabulate==0.9.0
Tempita==0.5.2
tenacity==4.12.0
termcolor==1.1.0
testrepository==0.0.20
tenacity==8.2.2
termcolor==2.3.0
testresources==2.0.1
testscenarios==0.5.0
testtools==2.5.0
text-unidecode==1.2
testtools==2.6.0
text-unidecode==1.3
thrift==0.16.0
thrift-sasl==0.4.3
tiddlyweb==2.4.3
tomli==2.0.1
tomlkit==0.11.8
tornado==5.1.1
typing-extensions==3.7.2
tzlocal==1.5.1
tornado==6.3.2
types-PyYAML==6.0.12.10
typing_extensions==4.6.3
tzdata==2023.3
uc-micro-py==1.0.2
ulid==1.1
unicodecsv==0.14.1
urllib3==1.25.11
uWSGI==2.0.21
vine==1.3.0
virtualenv==20.23.0
urllib3==1.26.16
uWSGI==2.0.22
vine==5.0.0
wcwidth==0.2.6
WebOb==1.8.7
websocket-client==1.5.1
Werkzeug==0.16.1
websocket-client==1.5.3
Werkzeug==2.2.3
wheel==0.40.0
wrapt==1.15.0
wsgi-intercept==1.11.0
WTForms==2.3.3
xmltodict==0.13.0
WTForms==3.0.1
yappi==1.4.0
yq==3.2.1
yarl==1.9.2
zipp==3.15.0
zope.deprecation==4.4.0

5
src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py
View File

@ -336,8 +336,9 @@ class ActionsResource(BaseResource):
# "conf" - JSON string that gets pickled into the DagRun's
# conf attribute. The conf is passed as as a string of escaped
# json inside the json payload accepted by the API.
conf_value = self.to_json({'action': action})
payload = {'run_id': action['id'], 'conf': conf_value}
# conf_value = self.to_json({'action': action})
payload = {'run_id': action['id'], 'conf': {'action': action}}
LOG.info('Request payload: %s', payload)
try:
resp = requests.post(req_url, timeout=(c_timeout, r_timeout),
headers=headers, json=payload)

4
src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_id_api.py
View File

@ -89,12 +89,12 @@ class ActionsIdResource(BaseResource):
action['notes'].append(note.view())
return action
def get_dag_run_by_id(self, dag_id, execution_date):
def get_dag_run_by_id(self, dag_id, run_id):
"""
Wrapper for call to the airflow db to get a dag_run
:returns: a dag run dictionary
"""
dag_run_list = self.get_dag_run_db(dag_id, execution_date)
dag_run_list = self.get_dag_run_db(dag_id, run_id)
# should be only one result, return the first one
if dag_run_list:
return dag_run_list[0]

2
src/bin/shipyard_airflow/shipyard_airflow/control/af_monitoring/workflows_api.py
View File

@ -76,7 +76,7 @@ class WorkflowIdResource(BaseResource):
"""
Retrieve a workflow by id,
:param helper: The WorkflowHelper constructed for this invocation
:param workflow_id: a string in {dag_id}__{execution_date} format
:param workflow_id: a string in {dag_id}__{run_id} format
identifying a workflow
:returns: a workflow detail dictionary including steps
"""

2
src/bin/shipyard_airflow/shipyard_airflow/control/start_shipyard.py
View File

@ -19,7 +19,7 @@ to the api startup to handle the Falcon specific setup.
import logging
from oslo_config import cfg
from werkzeug.contrib.profiler import ProfilerMiddleware
from werkzeug.middleware.profiler import ProfilerMiddleware
from shipyard_airflow.conf import config
import shipyard_airflow.control.api as api

15
src/bin/shipyard_airflow/shipyard_airflow/dags/samples/openstack_api_call.py
View File

@ -16,8 +16,14 @@
"""
import airflow
from airflow import DAG
from airflow.operators import OpenStackOperator
from datetime import timedelta
try:
from airflow.operators import OpenStackOperator
from config_path import config_path
except ImportError:
from shipyard_airflow.plugins.openstack_operators import \
OpenStackOperator
from shipyard_airflow.dags.config_path import config_path
default_args = {
@ -31,10 +37,11 @@ default_args = {
'retry_delay': timedelta(minutes=1),
}
dag = DAG('openstack_cli', default_args=default_args, schedule_interval=None)
dag = DAG('openstack_cli', default_args=default_args,
schedule_interval=None)
# Location of shiyard.conf
config_path = '/usr/local/airflow/plugins/shipyard.conf'
# # Location of shiyard.conf
# config_path = '/usr/local/airflow/plugins/shipyard.conf'
# Note that the shipyard.conf file needs to be placed on a volume
# that can be accessed by the containers

90
src/bin/shipyard_airflow/shipyard_airflow/db/airflow_db.py
View File

@ -37,9 +37,9 @@ class AirflowDbAccess(DbAccess):
SELECT
"id",
"dag_id",
"execution_date",
"state",
"run_id",
"execution_date",
"external_trigger",
"conf",
"end_date",
@ -52,9 +52,9 @@ class AirflowDbAccess(DbAccess):
SELECT
"id",
"dag_id",
"execution_date",
"state",
"run_id",
"execution_date",
"external_trigger",
"conf",
"end_date",
@ -73,9 +73,9 @@ class AirflowDbAccess(DbAccess):
SELECT
"id",
"dag_id",
"execution_date",
"state",
"run_id",
"execution_date",
"external_trigger",
"conf",
"end_date",
@ -93,6 +93,7 @@ class AirflowDbAccess(DbAccess):
"task_id",
"dag_id",
"execution_date",
"dr"."run_id",
"start_date",
"end_date",
"duration",
@ -109,7 +110,19 @@ class AirflowDbAccess(DbAccess):
"pid",
"max_tries"
FROM
task_instance
task_instance ti
INNER JOIN
(
SELECT
"execution_date",
"run_id"
FROM
dag_run
GROUP BY
run_id,
execution_date
) dr
ON ti.run_id=dr.run_id
ORDER BY
priority_weight desc,
start_date
@ -119,33 +132,48 @@ class AirflowDbAccess(DbAccess):
# used to merge into this query.
SELECT_TASKS_BY_ID = sqlalchemy.sql.text('''
SELECT
"task_id",
"dag_id",
"execution_date",
"start_date",
"end_date",
"duration",
"state",
"try_number",
"hostname",
"unixname",
"job_id",
"pool",
"queue",
"priority_weight",
"operator",
"queued_dttm",
"pid",
"max_tries"
FROM
task_instance
WHERE
dag_id LIKE :dag_id
AND
execution_date = :execution_date
ORDER BY
priority_weight desc,
start_date
"task_id",
"dag_id",
"execution_date",
"dr"."run_id",
"start_date",
"end_date",
"duration",
"state",
"try_number",
"hostname",
"unixname",
"job_id",
"pool",
"queue",
"priority_weight",
"operator",
"queued_dttm",
"pid",
"max_tries"
FROM
task_instance ti
INNER JOIN
(
SELECT
"execution_date",
"run_id"
FROM
dag_run
GROUP BY
run_id,
execution_date
) dr
ON
ti.run_id=dr.run_id
WHERE
dag_id LIKE :dag_id
AND
execution_date = :execution_date
ORDER BY
priority_weight desc,
start_date
LIMIT 1
''')
UPDATE_DAG_RUN_STATUS = sqlalchemy.sql.text('''

2
src/bin/shipyard_airflow/shipyard_airflow/db/common_db.py
View File

@ -53,7 +53,7 @@ class DbAccess:
"""
Unimplemented method for use in overriding to peform db updates
"""
LOG.info('No databse version updates specified for %s',
LOG.info('No database version updates specified for %s',
self.__class__.__name__)
def get_engine(self):

1
src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py
View File

@ -81,7 +81,6 @@ class ArmadaBaseOperator(UcpBaseOperator):
# Logs uuid of action performed by the Operator
LOG.info("Armada Operator for action %s", self.action_id)
# Set up armada client
self.armada_client = self._init_armada_client(
self.endpoints.endpoint_by_name(service_endpoint.ARMADA),

7
src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_create_site_action_tag.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import datetime
from datetime import timezone
import logging
# Using nosec to prevent Bandit blacklist reporting. Subprocess is used
# in a controlled way as part of this operator.
@ -43,7 +44,8 @@ class DeckhandCreateSiteActionTagOperator(DeckhandBaseOperator):
def do_execute(self):
# Calculate total elapsed time for workflow
time_delta = datetime.now() - self.task_instance.execution_date
time_delta = datetime.now(timezone.utc) \
- self.task_instance.execution_date
hours, remainder = divmod(time_delta.seconds, 3600)
minutes, seconds = divmod(remainder, 60)
@ -88,7 +90,8 @@ class DeckhandCreateSiteActionTagOperator(DeckhandBaseOperator):
# container.
response = subprocess.run( # nosec
['airflow',
'task_state',
'tasks',
'state',
self.main_dag_name,
task_id,
execution_date],

13
src/bin/shipyard_airflow/test-requirements.txt
View File

@ -1,13 +1,14 @@
bandit==1.6.0
flake8==3.8.4
pytest==3.5.0
pytest-cov==2.5.1
flake8
pytest==7.4.0
pytest-cov==4.1.0
coverage==5.3
testfixtures
tox<=4.0.0
typing-extensions==3.7.2
pyflakes>=2.1.1
# typing-extensions==3.7.2
pyflakes
# Pin apache-airflow 1.10.5 to align with version in airflow images requirements.txt
apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5
# apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5
# install postgres seperately as apache-airflow 1.10.5 forces postgres to use
# psycopg2 instead of psycopg2-binary which requires additional apt packages
# to be installed, i.e. postgres-devel. Otherwise the following error is seen:

6
src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group.py
View File

@ -236,12 +236,12 @@ class TestDeploymentGroup:
with pytest.raises(InvalidDeploymentGroupNodeLookupError) as err:
DeploymentGroup(yaml.safe_load(_GROUP_YAML_1),
broken_node_lookup_1)
assert str(err).endswith("is not an iterable")
assert err.match("is not an iterable")
with pytest.raises(InvalidDeploymentGroupNodeLookupError) as err:
DeploymentGroup(yaml.safe_load(_GROUP_YAML_1),
broken_node_lookup_2)
assert str(err).endswith("is not all strings")
assert err.match("is not all strings")
def test_set_stage(self):
dg = DeploymentGroup(yaml.safe_load(_GROUP_YAML_ALL_SELECTOR),
@ -269,7 +269,7 @@ class TestStage:
Stage.PREPARED]
with pytest.raises(DeploymentGroupStageError) as de:
Stage.previous_stage('Chickens and Turkeys')
assert str(de).endswith("Chickens and Turkeys is not a valid stage")
assert de.match("Chickens and Turkeys is not a valid stage")
class TestCheckLabelFormat:

18
src/bin/shipyard_airflow/tests/unit/control/af_monitoring/test_workflow_helper.py
View File

@ -92,7 +92,8 @@ def test__get_threshold_date():
arrow.get('2017-10-09T10:00:00.00000').naive
)
RUN_ID_ONE = "AAAAAAAAAAAAAAAAAAAAA"
RUN_ID_TWO = "BBBBBBBBBBBBBBBBBBBBB"
DATE_ONE = datetime(2017, 9, 13, 11, 13, 3, 57000)
DATE_TWO = datetime(2017, 9, 13, 11, 13, 5, 57000)
@ -100,7 +101,7 @@ DAG_RUN_1 = {
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'FAILED',
'run_id': '99',
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_ONE
@ -133,9 +134,8 @@ TASK_LIST = [
{
'task_id': '1a',
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'SUCCESS',
'run_id': '12345',
'run_id': RUN_ID_ONE,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_ONE,
@ -147,9 +147,8 @@ TASK_LIST = [
{
'task_id': '1b',
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'SUCCESS',
'run_id': '12345',
'run_id': RUN_ID_ONE,
'external_trigger': 'something',
'start_date': DATE_TWO,
'end_date': DATE_TWO,
@ -161,9 +160,8 @@ TASK_LIST = [
{
'task_id': '1c',
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'FAILED',
'run_id': '12345',
'run_id': RUN_ID_ONE,
'external_trigger': 'something',
'start_date': DATE_TWO,
'end_date': DATE_TWO,
@ -198,7 +196,7 @@ DAG_RUN_SUB = {
'dag_id': 'did2.didnt',
'execution_date': DATE_ONE,
'state': 'FAILED',
'run_id': '99',
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_ONE
@ -220,4 +218,4 @@ def test_get_workflow_subords():
assert dag_detail['dag_id'] == 'did2'
assert len(dag_detail['sub_dags']) == 1
assert dag_detail['sub_dags'][0]['dag_id'] == 'did2.didnt'
assert len(dag_detail['steps']) == 3
assert len(dag_detail['steps']) == 3

35
src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py
View File

@ -39,6 +39,8 @@ from shipyard_airflow.control.helpers.configdocs_helper import (
from shipyard_airflow.errors import ApiError
from shipyard_airflow.policy import ShipyardPolicy
RUN_ID_ONE = "AAAAAAAAAAAAAAAAAAAAA"
RUN_ID_TWO = "BBBBBBBBBBBBBBBBBBBBB"
DATE_ONE = datetime(2017, 9, 13, 11, 13, 3, 57000)
DATE_TWO = datetime(2017, 9, 13, 11, 13, 5, 57000)
DATE_ONE_STR = DATE_ONE.strftime('%Y-%m-%dT%H:%M:%S')
@ -107,7 +109,7 @@ def actions_db():
"""
return [
{
'id': 'aaaaaa',
'id': RUN_ID_ONE,
'name': 'dag_it',
'parameters': None,
'dag_id': 'did1',
@ -117,7 +119,7 @@ def actions_db():
'context_marker': '8-4-4-4-12a'
},
{
'id': 'bbbbbb',
'id': RUN_ID_TWO,
'name': 'dag2',
'parameters': {
'p1': 'p1val'
@ -140,7 +142,7 @@ def dag_runs_db():
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'SUCCESS',
'run_id': '12345',
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_TWO
@ -149,7 +151,7 @@ def dag_runs_db():
'dag_id': 'did1',
'execution_date': DATE_ONE,
'state': 'FAILED',
'run_id': '99',
'run_id': RUN_ID_ONE,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_ONE
@ -165,9 +167,9 @@ def tasks_db():
{
'task_id': '1a',
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'SUCCESS',
'run_id': '12345',
'execution_date': DATE_ONE,
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_TWO,
@ -179,9 +181,9 @@ def tasks_db():
{
'task_id': '1b',
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'SUCCESS',
'run_id': '12345',
'execution_date': DATE_ONE,
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_TWO,
@ -193,9 +195,9 @@ def tasks_db():
{
'task_id': '1c',
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'SUCCESS',
'run_id': '12345',
'execution_date': DATE_ONE,
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_TWO,
@ -207,8 +209,9 @@ def tasks_db():
{
'task_id': '2a',
'dag_id': 'did1',
'execution_date': DATE_ONE,
'state': 'FAILED',
'execution_date': DATE_ONE,
'run_id': RUN_ID_ONE,
'start_date': DATE_ONE,
'end_date': DATE_ONE,
'duration': '1second',
@ -354,16 +357,16 @@ def test_get_all_actions_notes(*args):
action_resource.get_all_dag_runs_db = dag_runs_db
action_resource.get_all_tasks_db = tasks_db
# inject some notes
nh.make_action_note('aaaaaa', "hello from aaaaaa1")
nh.make_action_note('aaaaaa', "hello from aaaaaa2")
nh.make_action_note('bbbbbb', "hello from bbbbbb")
nh.make_action_note(RUN_ID_ONE, "hello from aaaaaa1")
nh.make_action_note(RUN_ID_ONE, "hello from aaaaaa2")
nh.make_action_note(RUN_ID_TWO, "hello from bbbbbb")
result = action_resource.get_all_actions(verbosity=1)
assert len(result) == len(actions_db())
for action in result:
if action['id'] == 'aaaaaa':
if action['id'] == RUN_ID_ONE:
assert len(action['notes']) == 2
if action['id'] == 'bbbbbb':
if action['id'] == RUN_ID_TWO:
assert len(action['notes']) == 1
assert action['notes'][0]['note_val'] == 'hello from bbbbbb'

23
src/bin/shipyard_airflow/tests/unit/control/test_actions_id_api.py
View File

@ -28,6 +28,8 @@ from shipyard_airflow.db.db import AIRFLOW_DB, SHIPYARD_DB
from shipyard_airflow.errors import ApiError
from tests.unit.control.common import create_req, create_resp
RUN_ID_ONE = "AAAAAAAAAAAAAAAAAAAAA"
RUN_ID_TWO = "BBBBBBBBBBBBBBBBBBBBB"
DATE_ONE = datetime(2017, 9, 13, 11, 13, 3, 57000)
DATE_TWO = datetime(2017, 9, 13, 11, 13, 5, 57000)
DATE_ONE_STR = DATE_ONE.strftime('%Y-%m-%dT%H:%M:%S')
@ -50,10 +52,10 @@ def actions_db(action_id):
replaces the actual db call
"""
return {
'id': '12345678901234567890123456',
'id': RUN_ID_ONE,
'name': 'dag_it',
'parameters': None,
'dag_id': 'did2',
'dag_id': 'did1',
'dag_execution_date': DATE_ONE_STR,
'user': 'robot1',
'timestamp': DATE_ONE,
@ -69,7 +71,7 @@ def dag_runs_db(dag_id, execution_date):
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'FAILED',
'run_id': '99',
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_ONE
@ -83,9 +85,8 @@ def tasks_db(dag_id, execution_date):
return [{
'task_id': '1a',
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'SUCCESS',
'run_id': '12345',
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_ONE,
@ -96,9 +97,8 @@ def tasks_db(dag_id, execution_date):
}, {
'task_id': '1b',
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'SUCCESS',
'run_id': '12345',
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_TWO,
'end_date': DATE_TWO,
@ -109,9 +109,8 @@ def tasks_db(dag_id, execution_date):
}, {
'task_id': '1c',
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'FAILED',
'run_id': '12345',
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_TWO,
'end_date': DATE_TWO,
@ -230,7 +229,7 @@ def test_get_dag_run_by_id_notempty():
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'FAILED',
'run_id': '99',
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_ONE
@ -296,14 +295,13 @@ def test_get_tasks_db(mock_get_tasks_by_id):
dag_id=dag_id, execution_date=execution_date)
assert result == expected
@mock.patch.object(AIRFLOW_DB, 'get_dag_runs_by_id')
def test_get_dag_run_db(mock_get_dag_runs_by_id):
expected = {
'dag_id': 'did2',
'execution_date': DATE_ONE,
'state': 'FAILED',
'run_id': '99',
'run_id': RUN_ID_TWO,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_ONE
@ -318,7 +316,6 @@ def test_get_dag_run_db(mock_get_dag_runs_by_id):
dag_id=dag_id, execution_date=execution_date)
assert result == expected
@mock.patch.object(SHIPYARD_DB, 'get_command_audit_by_action_id')
def test_get_action_command_audit_db(mock_get_command_audit_by_action_id):
expected = {

15
src/bin/shipyard_airflow/tests/unit/control/test_actions_steps_id_api.py
View File

@ -26,6 +26,8 @@ from shipyard_airflow.control.action.actions_steps_id_api import \
from shipyard_airflow.errors import ApiError
from tests.unit.control import common
RUN_ID_ONE = "AAAAAAAAAAAAAAAAAAAAA"
RUN_ID_TWO = "BBBBBBBBBBBBBBBBBBBBB"
DATE_ONE = datetime(2017, 9, 13, 11, 13, 3, 57000)
DATE_TWO = datetime(2017, 9, 13, 11, 13, 5, 57000)
DATE_ONE_STR = DATE_ONE.strftime('%Y-%m-%dT%H:%M:%S')
@ -64,9 +66,9 @@ def tasks_db(dag_id, execution_date):
return [{
'task_id': '1a',
'dag_id': 'did2',
'execution_date': DATE_ONE,
# 'execution_date': DATE_ONE,
'state': 'SUCCESS',
'run_id': '12345',
'run_id': RUN_ID_ONE,
'external_trigger': 'something',
'start_date': DATE_ONE,
'end_date': DATE_ONE,
@ -77,9 +79,9 @@ def tasks_db(dag_id, execution_date):
}, {
'task_id': '1b',
'dag_id': 'did2',
'execution_date': DATE_ONE,
# 'execution_date': DATE_ONE,
'state': 'SUCCESS',
'run_id': '12345',
'run_id': RUN_ID_ONE,
'external_trigger': 'something',
'start_date': DATE_TWO,
'end_date': DATE_TWO,
@ -90,9 +92,9 @@ def tasks_db(dag_id, execution_date):
}, {
'task_id': '1c',
'dag_id': 'did2',
'execution_date': DATE_ONE,
# 'execution_date': DATE_ONE,
'state': 'FAILED',
'run_id': '12345',
'run_id': RUN_ID_ONE,
'external_trigger': 'something',
'start_date': DATE_TWO,
'end_date': DATE_TWO,
@ -172,3 +174,4 @@ class TestActionsStepsResource():
action_resource.get_tasks_db(dag_id, execution_date)
mock_get_tasks_by_id.assert_called_with(
dag_id=dag_id, execution_date=execution_date)

6
src/bin/shipyard_airflow/tests/unit/control/test_actions_steps_id_logs_api.py
View File

@ -57,7 +57,7 @@ TASK_INSTANCE_DB = [
{
'task_id': 'action_xcom',
'dag_id': 'deploy_site',
'execution_date': DATE_ONE,
# 'execution_date': DATE_ONE,
'start_date': DATE_TWO,
'end_date': DATE_THREE,
'duration': '4.353791',
@ -75,7 +75,7 @@ TASK_INSTANCE_DB = [
}, {
'task_id': 'dag_concurrency_check',
'dag_id': 'deploy_site',
'execution_date': DATE_ONE,
# 'execution_date': DATE_ONE,
'start_date': DATE_TWO,
'end_date': DATE_THREE,
'duration': '4.034112',
@ -93,7 +93,7 @@ TASK_INSTANCE_DB = [
}, {
'task_id': 'k8s_preflight_check',
'dag_id': 'deploy_site',
'execution_date': DATE_ONE,
# 'execution_date': DATE_ONE,
'start_date': DATE_TWO,
'end_date': DATE_THREE,
'duration': '4.452571',

2
src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py
View File

@ -39,7 +39,7 @@ RELEASES = {
}
class TestArmadaTestReleasesOperator:
class TestArmadaTestReleasesOperator(ArmadaTestReleasesOperator):
@mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info')
@mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True)
@mock.patch.object(ArmadaBaseOperator, 'get_releases',

5
src/bin/shipyard_airflow/tests/unit/plugins/test_drydock_destroy_nodes_operator.py
View File

@ -143,6 +143,7 @@ class TestDrydockDestroyNodesOperator:
assert ct.called
assert "Task destroy_nodes has failed." in caplog.text
@mock.patch.object(
DrydockDestroyNodeOperator, 'create_task'
)
@ -192,8 +193,8 @@ class TestDrydockDestroyNodesOperator:
assert gs.called
assert "Task destroy_nodes has timed out after 10 seconds." in (
caplog.text)
assert ("One or more nodes requested for destruction failed to "
"destroy") in str(ae.value)
assert ("One or more nodes requested for destruction failed to "
"destroy") in str(ae.value)
@mock.patch.object(
DrydockDestroyNodeOperator, 'get_successes_for_task',

10
src/bin/shipyard_airflow/tests/unit/plugins/test_drydock_nodes_operator.py
View File

@ -351,8 +351,9 @@ class TestDrydockNodesOperator:
assert op._execute_task.call_count == 1
@mock.patch("shipyard_airflow.plugins.check_k8s_node_status."
"check_node_status", return_value=[])
def test_execute_deployment(self, cns):
"check_node_status", return_value=[]
)
def test_execute_deployment(self,cns):
op = DrydockNodesOperator(main_dag_name="main",
shipyard_conf=CONF_FILE,
task_id="t1")
@ -369,8 +370,11 @@ class TestDrydockNodesOperator:
assert op._execute_task.call_count == 1
assert cns.call_count == 1
@mock.patch("shipyard_airflow.plugins.check_k8s_node_status."
"check_node_status", return_value=['node2', 'node4'])
"check_node_status",
return_value=['node2', 'node4']
)
def test_execute_deployment_k8s_fail(self, cns, caplog):
op = DrydockNodesOperator(main_dag_name="main",
shipyard_conf=CONF_FILE,

4
src/bin/shipyard_airflow/tests/unit/plugins/test_drydock_relabel_nodes_operator.py
View File

@ -187,8 +187,8 @@ class TestDrydockRelabelNodesOperator:
assert gs.called
assert "Task relabel_nodes has timed out after 2 seconds." in (
caplog.text)
assert ("One or more nodes requested for relabeling failed to "
"relabel") in str(ae.value)
assert ("One or more nodes requested for relabeling failed to "
"relabel") in str(ae.value)
@mock.patch.object(
DrydockRelabelNodesOperator, 'get_successes_for_task',

83
src/bin/shipyard_airflow/tox.ini
View File

@ -10,19 +10,6 @@ setenv=
deps =
-r{toxinidir}/requirements-frozen.txt
-r{toxinidir}/test-requirements.txt
[testenv:freeze-bionic]
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
commands=
rm -f {toxinidir}/requirements-frozen-bionic.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen-bionic.txt"
[testenv:freeze]
recreate = True
allowlist_externals=
@ -30,42 +17,54 @@ allowlist_externals=
sh
deps=
-r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
[testenv:py38]
skipsdist=True
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
basepython=python3.8
allowlist_externals=
bash
airflow
commands =
bash -c "rm -f $HOME/airflow/airflow.db"
pip install -r{toxinidir}/test-requirements.txt
airflow initdb
airflow run example_bash_operator runme_0 2018-01-01
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow dag_state example_bash_operator 2018-01-01
bash -c "rm -rf $HOME/airflow"
airflow version
airflow db init
airflow db upgrade
airflow info
airflow dags list
airflow dags list-import-errors
airflow tasks test example_bash_operator runme_0
airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow tasks run example_bash_operator runme_0 2018-01-01
airflow dags state example_bash_operator 2018-01-01
pytest {posargs} -vv
[testenv:cover]
skipsdist=True
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
PYTHONWARNINGS=ignore::DeprecationWarning,ignore::FutureWarning
allowlist_externals=
bash
airflow
commands =
bash -c "rm -f $HOME/airflow/airflow.db"
pip install -r{toxinidir}/test-requirements.txt
airflow initdb
airflow run example_bash_operator runme_0 2018-01-01
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow dag_state example_bash_operator 2018-01-01
bash -c "rm -rf $HOME/airflow"
airflow version
airflow db init
airflow db upgrade
airflow info
airflow dags list
airflow dags list-import-errors
airflow tasks test example_bash_operator runme_0
airflow dags backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow tasks run example_bash_operator runme_0 2018-01-01
airflow dags state example_bash_operator 2018-01-01
pytest \
{posargs} \
--cov-branch \
@ -77,30 +76,7 @@ commands =
--cov-report term \
-vv
[testenv:py36]
skipsdist=True
basepython=python3.6
deps =
-r{toxinidir}/requirements-frozen-bionic.txt
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
allowlist_externals=
bash
airflow
commands =
bash -c "rm -f $HOME/airflow/airflow.db"
pip install -r{toxinidir}/test-requirements.txt
airflow initdb
airflow run example_bash_operator runme_0 2018-01-01
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow dag_state example_bash_operator 2018-01-01
pytest {posargs} -vv
[testenv:pep8]
deps=
flake8>=3.3.0
bandit>=1.5.0
commands =
flake8 {toxinidir}/shipyard_airflow
bandit -r shipyard_airflow
@ -113,9 +89,6 @@ commands =
[testenv:genconfig]
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
deps =
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/requirements-frozen.txt
commands =
pip install . --use-pep517
oslo-config-generator --config-file=generator/config-generator.conf
@ -123,9 +96,6 @@ commands =
[testenv:genpolicy]
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
deps =
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/requirements-frozen.txt
commands =
pip install . --use-pep517
oslopolicy-sample-generator --config-file=generator/policy-generator.conf
@ -135,9 +105,6 @@ commands =
skipsdist=True
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
deps =
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/requirements-frozen.txt
commands =
pip install . --use-pep517
oslo-config-generator --config-file=generator/config-generator.conf

32
src/bin/shipyard_client/requirements-direct.txt
View File

@ -14,20 +14,18 @@
# CLI/Client requirements
arrow<=0.17.0
click==7.1.2
click-default-group==1.2
keystoneauth1<=5.1.1
requests==2.23.0
PTable==0.9.2
pylibyaml==0.1.0
PyYAML<=5.4.1
Flask==1.1.0
coverage==5.3
itsdangerous==2.0.1
Jinja2==2.10.3
MarkupSafe==2.0.1
pbr==5.5.1
python-dateutil==2.8.1
setuptools<=45.2.0
Werkzeug==0.16.1
arrow
click
click-default-group
keystoneauth1==4.3.1
requests
PTable
pylibyaml
PyYAML
Flask
itsdangerous
Jinja2
MarkupSafe
pbr
python-dateutil
Werkzeug

30
src/bin/shipyard_client/requirements-frozen-bionic.txt
View File

@ -1,30 +0,0 @@
arrow==0.17.0
certifi==2023.5.7
chardet==3.0.4
click==7.1.2
click-default-group==1.2
coverage==5.3
Flask==1.1.0
idna==2.10
importlib-metadata==4.8.3
iso8601==1.1.0
itsdangerous==2.0.1
Jinja2==2.10.3
keystoneauth1==5.1.1
MarkupSafe==2.0.1
os-service-types==1.7.0
pbr==5.5.1
pip==21.3.1
PTable==0.9.2
pylibyaml==0.1.0
python-dateutil==2.8.1
PyYAML==5.4.1
requests==2.23.0
setuptools==45.2.0
six==1.16.0
stevedore==3.5.2
typing_extensions==4.1.1
urllib3==1.25.11
Werkzeug==0.16.1
wheel==0.37.1
zipp==3.6.0

43
src/bin/shipyard_client/requirements-frozen.txt
View File

@ -1,27 +1,28 @@
arrow==0.17.0
arrow==1.2.3
certifi==2023.5.7
chardet==3.0.4
click==7.1.2
click-default-group==1.2
coverage==5.3
Flask==1.1.0
idna==2.10
iso8601==1.1.0
itsdangerous==2.0.1
Jinja2==2.10.3
keystoneauth1==5.1.1
MarkupSafe==2.0.1
charset-normalizer==3.1.0
click==8.1.3
click-default-group==1.2.2
Flask==2.2.5
idna==3.4
importlib-metadata==4.13.0
iso8601==2.0.0
itsdangerous==2.1.2
Jinja2==3.1.2
keystoneauth1==4.3.1
MarkupSafe==2.1.3
os-service-types==1.7.0
pbr==5.5.1
pip==23.0.1
pbr==5.11.1
pip==23.1.2
PTable==0.9.2
pylibyaml==0.1.0
python-dateutil==2.8.1
PyYAML==5.4.1
requests==2.23.0
setuptools==45.2.0
python-dateutil==2.8.2
PyYAML==6.0
requests==2.31.0
setuptools==67.7.2
six==1.16.0
stevedore==5.0.0
urllib3==1.25.11
Werkzeug==0.16.1
stevedore==5.1.0
urllib3==1.26.16
Werkzeug==2.2.3
wheel==0.40.0
zipp==3.15.0

14
src/bin/shipyard_client/test-requirements.txt
View File

@ -1,13 +1,13 @@
# Testing
pyflakes==2.2.0
amqp==2.6.1
pytest==3.5.0
pytest-cov==2.5.1
responses==0.10.2
testfixtures==5.1.1
pyflakes
pytest==7.4.0
pytest-cov==4.1.0
coverage==5.3
responses
testfixtures
# Linting
flake8==3.8.4
flake8
# Security scanning
bandit==1.6.0 # Apache-2.0

12
src/bin/shipyard_client/tests/unit/cli/describe/test_describe_actions.py
View File

@ -169,7 +169,6 @@ GET_STEP_API_RESP = """
"try_number": 1,
"task_id": "preflight",
"state": "success",
"execution_date": "2017-09-24 19:05:49",
"dag_id": "deploy_site",
"index": 1,
"start_date": "2017-09-24 19:05:59.281032",
@ -311,8 +310,7 @@ WF_API_RESP = """
"try_number": 1,
"state": "success",
"operator": "PythonOperator",
"dag_id": "deploy_site",
"execution_date": "2017-10-09 21:19:03"
"dag_id": "deploy_site"
},
{
"end_date": "2017-10-09 21:19:25.283785",
@ -323,8 +321,7 @@ WF_API_RESP = """
"try_number": 1,
"state": "success",
"operator": "ConcurrencyCheckOperator",
"dag_id": "deploy_site",
"execution_date": "2017-10-09 21:19:03"
"dag_id": "deploy_site"
},
{
"end_date": "2017-10-09 21:20:05.394677",
@ -335,8 +332,7 @@ WF_API_RESP = """
"try_number": 1,
"state": "failed",
"operator": "SubDagOperator",
"dag_id": "deploy_site",
"execution_date": "2017-10-09 21:19:03"
"dag_id": "deploy_site"
}
],
"dag_id": "deploy_site",
@ -344,7 +340,6 @@ WF_API_RESP = """
"run_id": "manual__2017-10-09T21:19:03",
"sub_dags": [
{
"execution_date": "2017-10-09 21:19:03",
"end_date": null,
"workflow_id": "deploy_site.preflight__2017-10-09T21:19:03.000000",
"start_date": "2017-10-09 21:19:35.082479",
@ -354,7 +349,6 @@ WF_API_RESP = """
"run_id": "backfill_2017-10-09T21:19:03"
},
{
"execution_date": "2017-10-09 21:19:03",
"end_date": null,
"workflow_id": "deploy_site.postflight__2017-10-09T21:19:03.000000",
"start_date": "2017-10-09 21:19:35.082479",

26
src/bin/shipyard_client/tox.ini
View File

@ -11,18 +11,6 @@ deps =
-r{toxinidir}/requirements-frozen.txt
-r{toxinidir}/test-requirements.txt
[testenv:freeze-bionic]
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
commands=
rm -f {toxinidir}/requirements-frozen-bionic.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen-bionic.txt"
[testenv:freeze]
recreate = True
allowlist_externals=
@ -30,20 +18,11 @@ allowlist_externals=
sh
deps=
-r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt
commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
[testenv:py36]
skipsdist=True
deps =
-r{toxinidir}/requirements-frozen-bionic.txt
-r{toxinidir}/test-requirements.txt
commands =
pytest \
{posargs} \
-vv
[testenv:py38]
skipsdist=True
commands =
@ -67,9 +46,6 @@ commands =
[testenv:pep8]
deps=
flake8>=3.3.0
bandit>=1.5.0
commands =
flake8 {toxinidir}/shipyard_client
bandit -r shipyard_airflow

53
tools/gate/playbooks/airskiff-deploy.yaml
View File

@ -23,10 +23,11 @@
tasks:
- name: Install Packaging python module for airship
- name: Install Packaging python module for tools/airship
block:
- pip:
name: packaging
version: 23.1
executable: pip3
become: True
@ -61,7 +62,7 @@
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
- name: Setup openstack client
- name: Setup OpenStack Client
shell: |
set -ex
./tools/deployment/airskiff/developer/020-setup-client.sh
@ -69,42 +70,35 @@
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
- name: Build all charts locally
shell: |
set -ex
./tools/deployment/airskiff/developer/015-make-all-charts.sh
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
- name: Build Shipyard and Airflow with submitted changes
shell: |
set -ex
export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
make images
docker system prune --force
args:
chdir: "{{ zuul.project.src_dir }}"
become: yes
- name: Use locally built images in manifests
shell: |
set -ex
export DISTRO={{ SHIPYARD_IMAGE_DISTRO }}
docker rm registry --force || true
docker run -d -p 5000:5000 --restart=always --name registry registry:2
if test "${DISTRO}" = 'ubuntu_bionic'
then
docker tag quay.io/airshipit/shipyard:latest-ubuntu_bionic localhost:5000/shipyard:latest-ubuntu_bionic
docker push localhost:5000/shipyard:latest-ubuntu_bionic
sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_bionic#g" ./site/airskiff/software/config/versions.yaml
sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_bionic#g" ./global/software/config/versions.yaml
else
docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal localhost:5000/shipyard:latest-ubuntu_focal
docker push localhost:5000/shipyard:latest-ubuntu_focal
sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml
sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./global/software/config/versions.yaml
fi
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
- name: Build all charts locally
shell: |
set -ex
./tools/deployment/airskiff/developer/015-make-all-charts.sh
docker tag quay.io/airshipit/airflow:latest-ubuntu_focal localhost:5000/airflow:latest-ubuntu_focal
docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal localhost:5000/shipyard:latest-ubuntu_focal
docker push localhost:5000/airflow:latest-ubuntu_focal
docker push localhost:5000/shipyard:latest-ubuntu_focal
sed -i "s#quay.io/airshipit/airflow:latest-ubuntu_focal#localhost:5000/airflow:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml
sed -i "s#quay.io/airshipit/shipyard:latest-ubuntu_focal#localhost:5000/shipyard:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
@ -121,8 +115,10 @@
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
- name: Deploy Airship components using Armada
shell: |
set -ex
mkdir -p ~/.kube
cp -rp /home/zuul/.kube/config ~/.kube/config
./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
@ -143,3 +139,12 @@
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
- name: Stop artifactory
shell: |
set -ex
# terminate artifactory
docker rm artifacts --force || true
args:
chdir: "{{ zuul.project.src_dir }}"
become: yes

2
tools/helm_install.sh
View File

@ -17,7 +17,7 @@
set -x
HELM=$1
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz"}
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"}
function install_helm_binary {

9
tox.ini
View File

@ -49,15 +49,6 @@ commands=
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample"
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample"
[testenv:py36]
deps =
tox<=4.0.0
allowlist_externals=
tox
commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py36
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py36
[testenv:cover]
deps =
tox<=4.0.0