From 154a099b2843b1b42a7b8afa5fa98383cf3c1b7c Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Fri, 10 Mar 2023 00:20:04 +0000 Subject: [PATCH] Shipyard upgrade for focal - upgraded Airflow to 1.10.15 - https://airflow.apache.org/docs/apache-airflow/1.10.15/changelog.html - disabled xenial, bionic and opensuse images build gates - added focal image build gate - added focal zuul build node - adjusted Makefile for focal - added bindep.txt to utilize bindep zuul base role for zuul build node pre-setup - added focal Dockerfile - implemented freeze requirements.txt approach like in other Airship projects - removed specific requirements.txt for airflow in favor of using requirements-frozen.txt from shipyard_airflow project when building airflow docker image - fixed docker image publishing to Quay - replaces deprecated LOG.warn with new LOG.warning call - replaced deprecated body attribute in responce wiht responce.text attribute - update of falcon module deprecated .API call - replaced wiht falcon.App call - deprecated routing.create_http_method_map method replaced with routing.map_http_methods - re-formatted code tabulations based on yapf recommendations - replaced deprecated protocol attribute in Pytest create_environ() with http_version attribute - replaced deprecated app attribute in Pytest create_environ() with root_path attribute - fixed airflow CLI commands to match 1.10.15 version - updated zuul gates to work on focal nodes and added focal specific node setup items by adding appriate ansible tasks and roles - uplifted Helm to 3.9.4 - uplifted stable HTK commit id - updated tox.in to with with tox v4 - uplifted dependences references to other Airship projects - common python dependences were syncronized with other Airship projects(Promenade, Deckhand, Armada, Drydock) - fixed airskiff deployment gate - fixed genconfig* profiles in shipyard-airflow tox.ini responsible for maintanance of policy.yaml.sample and shipyard.conf.sample Change-Id: I0c85187dc9bacf0849382563dd5ff7e9b2814c59 --- .zuul.yaml | 73 ++--- Makefile | 16 +- bindep.txt | 10 + doc/requirements.txt | 5 +- doc/source/_static/shipyard.conf.sample | 2 +- .../_static/shipyard.policy.yaml.sample | 1 - ....ubuntu_xenial => Dockerfile.ubuntu_focal} | 23 +- images/airflow/requirements.txt | 44 --- .../airflow/script/airflow_start_service.sh | 4 +- ....ubuntu_xenial => Dockerfile.ubuntu_focal} | 33 ++- .../etc/shipyard/shipyard.conf.sample | 55 +--- .../shipyard_airflow/requirements-direct.txt | 29 ++ .../shipyard_airflow/requirements-frozen.txt | 255 ++++++++++++++++++ src/bin/shipyard_airflow/requirements.txt | 64 +---- src/bin/shipyard_airflow/setup.cfg | 4 +- .../document_validators/document_validator.py | 10 +- .../common/notes/notes_helper.py | 4 +- .../control/action/actions_api.py | 4 +- .../control/action/actions_id_api.py | 2 +- .../control/action/actions_steps_id_api.py | 2 +- .../action/actions_steps_id_logs_api.py | 2 +- .../action/actions_validations_id_api.py | 2 +- .../control/af_monitoring/workflows_api.py | 4 +- .../shipyard_airflow/control/api.py | 4 +- .../shipyard_airflow/control/base.py | 2 +- .../control/configdocs/configdocs_api.py | 8 +- .../configdocs/rendered_configdocs_api.py | 2 +- .../control/middleware/common_params.py | 5 +- .../control/middleware/logging_mw.py | 2 +- .../control/notes/notedetails_api.py | 2 +- .../control/status/status_api.py | 2 +- .../validators/validate_deployment_action.py | 5 +- .../shipyard_airflow/errors.py | 2 +- .../plugins/drydock_base_operator.py | 23 +- .../plugins/drydock_destroy_nodes.py | 9 +- .../shipyard_airflow/plugins/drydock_nodes.py | 4 +- .../plugins/drydock_relabel_nodes.py | 9 +- .../plugins/drydock_validate_design.py | 3 +- .../plugins/promenade_validate_site_design.py | 3 +- .../shipyard_airflow/test-requirements.txt | 23 +- .../tests/unit/control/common.py | 4 +- .../tests/unit/control/test_actions_api.py | 8 +- .../tests/unit/control/test_actions_id_api.py | 2 +- .../control/test_actions_steps_id_logs_api.py | 2 +- .../tests/unit/control/test_api.py | 2 +- .../tests/unit/control/test_configdocs_api.py | 2 +- .../tests/unit/control/test_status_api.py | 2 +- src/bin/shipyard_airflow/tox.ini | 45 +++- .../shipyard_client/requirements-direct.txt | 24 ++ .../shipyard_client/requirements-frozen.txt | 40 +++ src/bin/shipyard_client/requirements.txt | 27 +- src/bin/shipyard_client/setup.cfg | 4 +- src/bin/shipyard_client/test-requirements.txt | 8 +- src/bin/shipyard_client/tox.ini | 24 +- tools/airflow_image_run.sh | 2 +- tools/gate/playbooks/airskiff-deploy.yaml | 39 ++- .../gate/playbooks/airskiff-reduce-site.yaml | 30 --- tools/gate/playbooks/run-image.yaml | 10 + .../build-images/files/docker-systemd.conf | 8 - .../build-images/tasks/airship-shipyard.yaml | 43 +-- .../tasks/disable-systemd-resolved.yaml | 37 +++ .../disable-systemd-resolved/tasks/main.yaml | 15 ++ tools/helm_install.sh | 2 +- tools/helm_tk.sh | 4 +- tox.ini | 24 +- 65 files changed, 686 insertions(+), 478 deletions(-) create mode 100644 bindep.txt rename images/airflow/{Dockerfile.ubuntu_xenial => Dockerfile.ubuntu_focal} (90%) delete mode 100644 images/airflow/requirements.txt rename images/shipyard/{Dockerfile.ubuntu_xenial => Dockerfile.ubuntu_focal} (81%) create mode 100644 src/bin/shipyard_airflow/requirements-direct.txt create mode 100644 src/bin/shipyard_airflow/requirements-frozen.txt create mode 100644 src/bin/shipyard_client/requirements-direct.txt create mode 100644 src/bin/shipyard_client/requirements-frozen.txt delete mode 100644 tools/gate/roles/build-images/files/docker-systemd.conf create mode 100644 tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml create mode 100644 tools/gate/roles/disable-systemd-resolved/tasks/main.yaml diff --git a/.zuul.yaml b/.zuul.yaml index f6dcb97e..aea9a16a 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -13,7 +13,7 @@ - project: templates: - docs-on-readthedocs - - openstack-python36-jobs + - openstack-python38-jobs vars: rtd_webhook_id: '38576' rtd_project_name: 'airship-shipyard' @@ -25,22 +25,16 @@ - airship-shipyard-chart-build-latest-htk - airship-shipyard-whitespace-lint-gate - airship-shipyard-airskiff-deployment - - airship-shipyard-image-gate-ubuntu_xenial - - airship-shipyard-image-gate-ubuntu_bionic - - airship-shipyard-image-gate-opensuse + - airship-shipyard-image-gate-ubuntu_focal gate: jobs: - openstack-tox-pep8 - airship-shipyard-chart-build-gate - airship-shipyard-whitespace-lint-gate - - airship-shipyard-image-gate-ubuntu_xenial - - airship-shipyard-image-gate-ubuntu_bionic - - airship-shipyard-image-gate-opensuse + - airship-shipyard-image-gate-ubuntu_focal post: jobs: - - airship-shipyard-docker-build-post-ubuntu_xenial - - airship-shipyard-docker-build-post-ubuntu_bionic - - airship-shipyard-docker-build-post-opensuse + - airship-shipyard-docker-build-post-ubuntu_focal - shipyard-upload-git-mirror - nodeset: @@ -49,12 +43,18 @@ - name: primary label: ubuntu-bionic +- nodeset: + name: airship-shipyard-single-node-focal + nodes: + - name: primary + label: ubuntu-focal + - job: name: airship-shipyard-chart-build-gate description: Build charts using pinned Helm toolkit. timeout: 900 run: tools/gate/playbooks/build-charts.yaml - nodeset: airship-shipyard-single-node + nodeset: airship-shipyard-single-node-focal - job: name: airship-shipyard-chart-build-latest-htk @@ -62,7 +62,7 @@ timeout: 900 voting: false run: tools/gate/playbooks/build-charts.yaml - nodeset: airship-shipyard-single-node + nodeset: airship-shipyard-single-node-focal vars: HTK_COMMIT: master @@ -71,7 +71,7 @@ description: | Lints all files by checking them for whitespace. run: tools/gate/playbooks/zuul-linter.yaml - nodeset: airship-shipyard-single-node + nodeset: airship-shipyard-single-node-focal - job: name: airship-shipyard-image-base @@ -79,44 +79,31 @@ Base job for running airship-shipyard image related jobs. timeout: 3600 run: tools/gate/playbooks/run-image.yaml - nodeset: airship-shipyard-single-node + nodeset: airship-shipyard-single-node-focal irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - ^releasenotes/.*$ - job: - name: airship-shipyard-image-gate-ubuntu_xenial - voting: false + name: airship-shipyard-image-gate-ubuntu_focal description: | - Run shipyard-image build for ubuntu_xenial. + Run shipyard-image build for ubuntu_focal. parent: airship-shipyard-image-base vars: publish: false - distro: ubuntu_xenial - tags: - dynamic: - patch_set: true - -- job: - name: airship-shipyard-image-gate-ubuntu_bionic - description: | - Run shipyard-image build for ubuntu_bionic. - parent: airship-shipyard-image-base - vars: - publish: false - distro: ubuntu_bionic + distro: ubuntu_focal tags: dynamic: patch_set: true - job: name: airship-shipyard-airskiff-deployment - nodeset: airship-shipyard-single-node + nodeset: airship-shipyard-single-node-focal description: | Deploy Memcached using Airskiff and submitted Armada changes. timeout: 9600 - voting: false + voting: true pre-run: - tools/gate/playbooks/airskiff-reduce-site.yaml - tools/gate/playbooks/git-config.yaml @@ -144,27 +131,9 @@ dynamic: patch_set: true -- job: - name: airship-shipyard-docker-build-post-ubuntu_xenial - voting: false - timeout: 1800 - run: tools/gate/playbooks/run-image.yaml - nodeset: airship-shipyard-single-node - irrelevant-files: *irrelevant-files - secrets: - - airship_shipyard_quay_creds - vars: - publish: true - distro: ubuntu_xenial - tags: - dynamic: - branch: true - commit: true - static: - - latest - job: - name: airship-shipyard-docker-build-post-ubuntu_bionic + name: airship-shipyard-docker-build-post-ubuntu_focal timeout: 1800 run: tools/gate/playbooks/run-image.yaml nodeset: airship-shipyard-single-node @@ -173,7 +142,7 @@ - airship_shipyard_quay_creds vars: publish: true - distro: ubuntu_bionic + distro: ubuntu_focal tags: dynamic: branch: true diff --git a/Makefile b/Makefile index afd46737..9dd33cf0 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ USE_PROXY ?= false AIRFLOW_SRC ?= AIRFLOW_HOME ?= DISTRO_BASE_IMAGE ?= -DISTRO ?= ubuntu_bionic +DISTRO ?= ubuntu_focal IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO} IMAGE_DIR:=images/$(IMAGE_NAME) @@ -61,7 +61,7 @@ charts: clean helm-toolkit # Perform Linting .PHONY: lint -lint: pep8 helm_lint build_docs +lint: pep8 helm-lint build_docs # Dry run templating of chart .PHONY: dry-run @@ -129,20 +129,26 @@ clean: rm -rf doc/build cd $(BUILD_CTX)/shipyard_client; rm -rf build cd $(BUILD_CTX)/shipyard_airflow; rm -rf build + rm -rf doc/build + rm -f charts/*.tgz + rm -f charts/*/requirements.lock + rm -rf charts/*/charts + rm -rf .tox .PHONY: pep8 pep8: cd $(BUILD_CTX)/shipyard_client; tox -e pep8 cd $(BUILD_CTX)/shipyard_airflow; tox -e pep8 -.PHONY: helm_lint -helm_lint: clean helm-toolkit +.PHONY: helm-lint +helm-lint: clean helm-toolkit + $(HELM) dep up charts/shipyard $(HELM) lint charts/shipyard # Initialize local helm config .PHONY: helm-toolkit helm-toolkit: helm-install - tools/helm_tk.sh $(HELM) + ./tools/helm_tk.sh $(HELM) # Install helm binary .PHONY: helm-install diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 00000000..180ff05e --- /dev/null +++ b/bindep.txt @@ -0,0 +1,10 @@ +# This is a cross-platform list tracking distribution packages needed by tests; +# see http://docs.openstack.org/infra/bindep/ for additional information. + +libffi-dev [test platform:dpkg] +libkrb5-dev [platform:dpkg] +libpq-dev [platform:dpkg] +libsasl2-dev [platform:dpkg] +libssl-dev [platform:dpkg] +libre2-dev [platform:dpkg] +libzmq3-dev [platform:dpkg] \ No newline at end of file diff --git a/doc/requirements.txt b/doc/requirements.txt index 1f5d803d..b16a479b 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,5 +1,6 @@ # # Requirements for creating documentation only. # -sphinx>=1.6.2 -sphinx_rtd_theme==0.2.4 +Sphinx +sphinx-rtd-theme==0.5.0 +Jinja2<=3.0.3 \ No newline at end of file diff --git a/doc/source/_static/shipyard.conf.sample b/doc/source/_static/shipyard.conf.sample index 03e1e457..e9be4c5a 100644 --- a/doc/source/_static/shipyard.conf.sample +++ b/doc/source/_static/shipyard.conf.sample @@ -472,4 +472,4 @@ # Warning - Print a Warning level message if the validation fails # Error - Return an error when the validation fails and prevent the commit from # proceeding -#deployment_version_commit = Skip +#deployment_version_commit = Skip \ No newline at end of file diff --git a/doc/source/_static/shipyard.policy.yaml.sample b/doc/source/_static/shipyard.policy.yaml.sample index 63a4c9c7..20293810 100644 --- a/doc/source/_static/shipyard.policy.yaml.sample +++ b/doc/source/_static/shipyard.policy.yaml.sample @@ -106,4 +106,3 @@ # targeted release # POST /api/v1.0/actions #"workflow_orchestrator:action_test_site": "rule:admin_required" - diff --git a/images/airflow/Dockerfile.ubuntu_xenial b/images/airflow/Dockerfile.ubuntu_focal similarity index 90% rename from images/airflow/Dockerfile.ubuntu_xenial rename to images/airflow/Dockerfile.ubuntu_focal index 788777c8..e3f1572c 100644 --- a/images/airflow/Dockerfile.ubuntu_xenial +++ b/images/airflow/Dockerfile.ubuntu_focal @@ -13,7 +13,7 @@ # limitations under the License. # Docker image to run Airflow on Kubernetes -ARG FROM=ubuntu:16.04 +ARG FROM=ubuntu:focal FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ @@ -43,12 +43,12 @@ ARG AIRFLOW_HOME=/usr/local/airflow # Moved celery to images/airflow/requirements.txt as apache-airflow uses a # version of celery incompatibile with the version of kombu needed by other # Airship components -ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5" +ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs]==1.10.15" ARG DEBIAN_FRONTEND=noninteractive ARG ctx_base=src/bin # Kubectl version -ARG KUBECTL_VERSION=1.24.6 +ARG KUBECTL_VERSION=1.26.3 # Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated ENV AIRFLOW_HOME=${AIRFLOW_HOME} @@ -61,22 +61,23 @@ RUN set -ex && \ curl \ git \ g++ \ + libkrb5-dev \ libffi-dev \ libssl-dev \ libpq-dev \ libtool \ + libsasl2-dev \ locales \ netcat \ netbase \ python3 \ + python3-setuptools \ python3-pip \ python3-dev \ python3-dateutil \ - python3-openssl \ make \ --no-install-recommends \ - && python3 -m pip install -U 'pip<21.0' \ - && python3 -m pip install -U 'setuptools<58' \ + && python3 -m pip install -U pip \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ @@ -111,9 +112,9 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \ # Note - removing snakebite (python 2 vs. 3). See: # https://github.com/puckel/docker-airflow/issues/77 # Install Airflow directly to allow overriding source -COPY images/airflow/requirements.txt /tmp/ -RUN sed -i "s/pyOpenSSL==.*//" /tmp/requirements.txt \ - && pip3 install -r /tmp/requirements.txt --no-cache-dir \ +# COPY images/airflow/requirements.txt /tmp/ +COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/requirements.txt +RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \ && pip3 install $AIRFLOW_SRC --no-cache-dir \ && (pip3 uninstall -y snakebite || true) \ && (pip3 uninstall -y psycopg2 || true) \ @@ -137,12 +138,12 @@ ENV PBR_VERSION 0.1a1 # Since Shipyard and Airflow are built together as images, this should prevent # stale or out-of-date code between these parts. # Shipyard requirements, source and installation -COPY ${ctx_base}/shipyard_airflow/requirements.txt /tmp/api_requirements.txt +COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/ RUN cd /tmp/shipyard \ - && python3 setup.py install + && pip3 install $(pwd) # Note: The value for the dags and plugins directories that are sourced # from the values.yaml of the Shipyard Helm chart need to align with these diff --git a/images/airflow/requirements.txt b/images/airflow/requirements.txt deleted file mode 100644 index c172a45d..00000000 --- a/images/airflow/requirements.txt +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -celery==4.3 -pytz==2018.5 -python-dateutil==2.8.1 -pyOpenSSL==22.0.0 -ndg-httpsclient==0.5.1 -oslo.utils==3.42.1 -oslo.config==7.0.0 -oslo.serialization==2.29.2 -pyasn1<0.5.0,>=0.4.6 -psycopg2-binary==2.8.4 -docker==3.7.2 -# Airflow is now installed in the Dockerfile directory to allow for -# overriding where it is sourced from -python-openstackclient==3.16.1 -python-keystoneclient==3.22.0 -kubernetes>=6.0.0 -# Need to lock marshmellow-sqlalchemy and tabulate for compatibility issues -marshmallow-sqlalchemy==0.18.0 -WTForms~=2.2.1 -tabulate==0.8.03 -pbr==5.4.5 -setuptools>=40.4.3,<58 -six>=1.15.0 -urllib3==1.25.9 -vine<5.0.0a1,>=1.1.3 - -# Dependencies for other UCP components -git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65edada7d386cd55d2a#egg=deckhand -git+https://opendev.org/airship/drydock.git@96db568d4b28f5f163efbf606854243614a6c821#egg=drydock_provisioner&subdirectory=python -git+https://opendev.org/airship/armada.git@416eff4e52f72169c3ebf49a8ffca36086692b7a#egg=armada diff --git a/images/airflow/script/airflow_start_service.sh b/images/airflow/script/airflow_start_service.sh index 02a83a54..56992ece 100755 --- a/images/airflow/script/airflow_start_service.sh +++ b/images/airflow/script/airflow_start_service.sh @@ -41,9 +41,9 @@ elif [[ $cmd == *scheduler* ]]; then elif [[ $cmd == 'quicktest' ]]; then ${python3_path} ${airflow_path} initdb ${python3_path} ${airflow_path} webserver -p 8080 & - airflow run example_bash_operator runme_0 2018-01-01 + airflow tasks run example_bash_operator runme_0 2018-01-01 airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02 - airflow dag_state example_bash_operator 2018-01-01 + airflow dags state example_bash_operator 2018-01-01 else echo "Invalid Command!" exit 1 diff --git a/images/shipyard/Dockerfile.ubuntu_xenial b/images/shipyard/Dockerfile.ubuntu_focal similarity index 81% rename from images/shipyard/Dockerfile.ubuntu_xenial rename to images/shipyard/Dockerfile.ubuntu_focal index e0c0be23..fc91620e 100644 --- a/images/shipyard/Dockerfile.ubuntu_xenial +++ b/images/shipyard/Dockerfile.ubuntu_focal @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG FROM=ubuntu:16.04 +ARG FROM=ubuntu:focal FROM ${FROM} LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ @@ -41,11 +41,13 @@ RUN set -ex && \ automake \ ca-certificates \ curl \ + build-essential \ git \ libtool \ make \ netbase \ python3-dev \ + python3-setuptools \ --no-install-recommends \ && apt-get autoremove -yqq --purge \ && apt-get clean \ @@ -84,33 +86,36 @@ RUN chown -R shipyard: /home/shipyard \ && chmod +x /home/shipyard/entrypoint.sh # Requirements and Shipyard source -COPY ${ctx_base}/shipyard_airflow/requirements.txt /home/shipyard/api_requirements.txt -COPY ${ctx_base}/shipyard_client/requirements.txt /home/shipyard/client_requirements.txt +COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /home/shipyard/api_requirements.txt +COPY ${ctx_base}/shipyard_client/requirements-frozen.txt /home/shipyard/client_requirements.txt COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/ # Build RUN set -ex \ && buildDeps=' \ - gcc \ + build-essential \ git \ libssl-dev \ + libffi-dev \ + libkrb5-dev \ + libpq-dev \ + libsasl2-dev \ + libssl-dev \ + libre2-dev \ + libzmq3-dev \ make \ python3-pip \ ' \ && apt-get -qq update \ - && apt-get -y install -y \ - $buildDeps \ - python3-openssl \ - libffi-dev \ - python3-dev \ - --no-install-recommends \ - && python3 -m pip install -U 'pip<21.0' \ - && python3 -m pip install -U setuptools \ - && sed -i "s/pyOpenSSL==.*//" /home/shipyard/client_requirements.txt\ + && apt-get -y install -y $buildDeps --no-install-recommends \ + && python3 -m pip install -U pip \ + && pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \ + && cd /home/shipyard/shipyard_client \ + && pip3 install $(pwd) --use-pep517 \ && pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \ && cd /home/shipyard/shipyard \ - && python3 setup.py install \ + && pip3 install $(pwd) --use-pep517 \ && apt-get purge -y --auto-remove $buildDeps \ && apt-get autoremove -yqq --purge \ && apt-get clean \ diff --git a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample index 03e1e457..a822dead 100644 --- a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample +++ b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample @@ -180,9 +180,13 @@ # and will be removed in the S release. #auth_uri = -# API version of the admin Identity API endpoint. (string value) +# API version of the Identity API endpoint. (string value) #auth_version = +# Interface to use for the Identity API endpoint. Valid values are "public", +# "internal" (default) or "admin". (string value) +#interface = internal + # Do not handle authorization requests within the middleware, but delegate the # authorization decision to downstream WSGI components. (boolean value) #delay_auth_decision = false @@ -217,14 +221,6 @@ # The region in which the identity server can be found. (string value) #region_name = -# DEPRECATED: Directory used to cache files related to PKI tokens. This option -# has been deprecated in the Ocata release and will be removed in the P -# release. (string value) -# This option is deprecated for removal since Ocata. -# Its value may be silently ignored in the future. -# Reason: PKI token format is no longer supported. -#signing_dir = - # Optionally specify a list of memcached server(s) to use for caching. If left # undefined, tokens will instead be cached in-process. (list value) # Deprecated group/name - [keystone_authtoken]/memcache_servers @@ -235,16 +231,6 @@ # to -1 to disable caching completely. (integer value) #token_cache_time = 300 -# DEPRECATED: Determines the frequency at which the list of revoked tokens is -# retrieved from the Identity service (in seconds). A high number of revocation -# events combined with a low cache duration may significantly reduce -# performance. Only valid for PKI tokens. This option has been deprecated in -# the Ocata release and will be removed in the P release. (integer value) -# This option is deprecated for removal since Ocata. -# Its value may be silently ignored in the future. -# Reason: PKI token format is no longer supported. -#revocation_cache_time = 10 - # (Optional) If defined, indicate whether token data should be authenticated or # authenticated and encrypted. If MAC, token data is authenticated (with HMAC) # in the cache. If ENCRYPT, token data is encrypted and authenticated in the @@ -280,9 +266,9 @@ # client connection from the pool. (integer value) #memcache_pool_conn_get_timeout = 10 -# (Optional) Use the advanced (eventlet safe) memcached client pool. The -# advanced pool will only work under python 2.x. (boolean value) -#memcache_use_advanced_pool = false +# (Optional) Use the advanced (eventlet safe) memcached client pool. (boolean +# value) +#memcache_use_advanced_pool = true # (Optional) Indicate whether to set the X-Service-Catalog header. If False, # middleware will not ask for service catalog on token validation and will not @@ -298,27 +284,6 @@ # (string value) #enforce_token_bind = permissive -# DEPRECATED: If true, the revocation list will be checked for cached tokens. -# This requires that PKI tokens are configured on the identity server. (boolean -# value) -# This option is deprecated for removal since Ocata. -# Its value may be silently ignored in the future. -# Reason: PKI token format is no longer supported. -#check_revocations_for_cached = false - -# DEPRECATED: Hash algorithms to use for hashing PKI tokens. This may be a -# single algorithm or multiple. The algorithms are those supported by Python -# standard hashlib.new(). The hashes will be tried in the order given, so put -# the preferred one first for performance. The result of the first hash will be -# stored in the cache. This will typically be set to multiple values only while -# migrating from a less secure algorithm to a more secure one. Once all the old -# tokens are expired this option should be set to a single value for better -# performance. (list value) -# This option is deprecated for removal since Ocata. -# Its value may be silently ignored in the future. -# Reason: PKI token format is no longer supported. -#hash_algorithms = md5 - # A choice of roles that must be present in a service token. Service tokens are # allowed to request that an expired token can be used and so this check should # tightly control that only actual services should be sending this token. Roles @@ -333,6 +298,10 @@ # possible. (boolean value) #service_token_roles_required = false +# The name or type of the service as it appears in the service catalog. This is +# used to validate tokens that have restricted access rules. (string value) +#service_type = + # Authentication type to load (string value) # Deprecated group/name - [keystone_authtoken]/auth_plugin #auth_type = diff --git a/src/bin/shipyard_airflow/requirements-direct.txt b/src/bin/shipyard_airflow/requirements-direct.txt new file mode 100644 index 00000000..88cde87e --- /dev/null +++ b/src/bin/shipyard_airflow/requirements-direct.txt @@ -0,0 +1,29 @@ +arrow<=0.17.0 +# apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.15 +apache-airflow[crypto,celery,hive,hdfs]==1.10.15 +coverage==5.3 +falcon +jsonschema<=3.2.0 +keystoneauth1<=5.1.1 +kubernetes==26.1.0 +networkx +oslo.config<=8.7.1 +oslo.policy<=3.10.1 +oslo.utils<=4.12.3 +pyarrow==6.0.1 +pylibyaml==0.1.0 +PyYAML<=5.4.1 +regex==2020.11.13 +requests==2.23.0 +responses==0.12.1 +setuptools<=45.2.0 +SQLAlchemy<=1.3.20 +ulid==1.1 +uWSGI==2.0.21 +Werkzeug +WTForms<=2.3.3 + +git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand +git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#egg=drydock_provisioner&subdirectory=python +git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4#egg=armada +git+https://opendev.org/airship/promenade.git@5602e85fb936eeafdc3e4699fd3afbe1576078bc#egg=promenade diff --git a/src/bin/shipyard_airflow/requirements-frozen.txt b/src/bin/shipyard_airflow/requirements-frozen.txt new file mode 100644 index 00000000..2edb4a5d --- /dev/null +++ b/src/bin/shipyard_airflow/requirements-frozen.txt @@ -0,0 +1,255 @@ +alabaster==0.7.13 +alembic==1.4.3 +amqp==2.6.1 +apache-airflow==1.10.15 +apispec==1.3.3 +argcomplete==1.12.3 +Armada @ git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4#egg=armada +arrow==0.17.0 +attrs==20.3.0 +autopage==0.5.1 +Babel==2.12.1 +bandit==1.6.0 +bcrypt==4.0.1 +Beaker==1.12.0 +billiard==3.6.4.0 +cached-property==1.5.2 +cachetools==5.3.0 +cattrs==1.10.0 +celery==4.4.7 +certifi==2022.12.7 +cffi==1.15.1 +chardet==3.0.4 +charset-normalizer==3.1.0 +click==7.1.2 +cliff==3.10.1 +cmd2==2.4.3 +colorama==0.4.6 +colorlog==4.0.2 +configparser==3.5.3 +coverage==5.3 +croniter==0.3.37 +cryptography==3.4.8 +debtcollector==2.5.0 +Deckhand @ git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand +decorator==5.1.1 +deepdiff==5.8.1 +defusedxml==0.7.1 +dill==0.3.6 +distlib==0.3.6 +dnspython==2.3.0 +docutils==0.17 +dogpile.cache==1.2.0 +drydock-provisioner @ git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#egg=drydock_provisioner&subdirectory=python +dulwich==0.21.3 +email-validator==1.3.1 +eventlet==0.33.3 +extras==1.0.0 +falcon==3.1.1 +fasteners==0.18 +filelock==3.12.0 +fixtures==3.0.0 +flake8==3.8.4 +Flask==1.1.4 +Flask-Admin==1.5.4 +Flask-AppBuilder==2.3.4 +Flask-Babel==1.0.0 +Flask-Caching==1.3.3 +Flask-JWT-Extended==3.25.1 +Flask-Login==0.4.1 +Flask-OpenID==1.3.0 +Flask-SQLAlchemy==2.5.1 +flask-swagger==0.2.14 +Flask-WTF==0.14.3 +flower==0.9.7 +funcsigs==1.0.2 +future==0.18.3 +futurist==2.4.1 +gitdb==4.0.10 +GitPython==3.1.31 +google-auth==2.17.3 +graphviz==0.20.1 +greenlet==2.0.2 +gunicorn==20.1.0 +hacking==4.1.0 +hmsclient==0.1.1 +html5lib==0.9999999 +httpexceptor==1.4.0 +humanize==4.6.0 +idna==2.10 +imagesize==1.4.1 +importlib-metadata==2.1.3 +importlib-resources==1.5.0 +iso8601==1.1.0 +itsdangerous==1.1.0 +Jinja2==2.11.3 +json-merge-patch==0.2 +jsonpath-ng==1.5.3 +jsonpath-rw==1.4.0 +jsonpath-rw-ext==1.2.2 +jsonpickle==1.4.1 +jsonschema==3.2.0 +keystoneauth1==5.1.1 +keystonemiddleware==10.2.0 +kombu==4.6.11 +kubernetes==26.1.0 +lazy-object-proxy==1.4.3 +lockfile==0.12.2 +Mako==1.2.4 +Markdown==2.6.11 +MarkupSafe==2.0.1 +marshmallow==2.21.0 +marshmallow-enum==1.5.1 +marshmallow-sqlalchemy==0.23.1 +mccabe==0.6.1 +mock==5.0.2 +more-itertools==9.1.0 +msgpack==1.0.5 +natsort==8.3.1 +netaddr==0.8.0 +netifaces==0.11.0 +networkx==3.1 +nose==1.3.7 +numpy==1.24.3 +oauthlib==3.2.2 +ordered-set==4.1.0 +os-service-types==1.7.0 +oslo.cache==2.10.1 +oslo.concurrency==5.1.1 +oslo.config==8.7.1 +oslo.context==4.1.0 +oslo.db==10.0.0 +oslo.i18n==6.0.0 +oslo.log==4.6.0 +oslo.messaging==12.13.0 +oslo.metrics==0.6.0 +oslo.middleware==4.4.0 +oslo.policy==3.10.1 +oslo.serialization==4.2.0 +oslo.service==3.1.1 +oslo.utils==4.12.3 +packaging==21.3 +pandas==1.5.3 +Paste==3.5.0 +PasteDeploy==3.0.1 +PasteScript==3.3.0 +pbr==5.5.1 +pendulum==1.4.4 +pip==23.0.1 +platformdirs==3.4.0 +pluggy==0.6.0 +ply==3.11 +prettytable==3.7.0 +prison==0.2.1 +promenade @ git+https://opendev.org/airship/promenade.git@5602e85fb936eeafdc3e4699fd3afbe1576078bc#egg=promenade +prometheus-client==0.8.0 +protobuf==4.22.3 +psutil==5.9.5 +psycopg2-binary==2.9.6 +pure-sasl==0.6.2 +py==1.11.0 +pyarrow==6.0.1 +pyasn1==0.5.0 +pyasn1-modules==0.3.0 +pycadf==3.1.1 +pycodestyle==2.6.0 +pycparser==2.21 +pyflakes==2.2.0 +Pygments==2.14.0 +PyHive==0.6.5 +PyJWT==1.7.1 +pylibyaml==0.1.0 +pymongo==4.3.3 +pyparsing==2.4.7 +pyperclip==1.8.2 +pyproject_api==1.5.0 +pyrsistent==0.19.3 +pytest==3.5.0 +pytest-cov==2.5.1 +python-barbicanclient==5.2.0 +python-daemon==2.3.2 +python-dateutil==2.8.1 +python-editor==1.0.4 +python-keystoneclient==5.1.0 +python-memcached==1.59 +python-mimeparse==1.6.0 +python-nvd3==0.15.0 +python-slugify==4.0.1 +python-subunit==1.4.0 +python3-openid==3.2.0 +pytz==2023.3 +pytzdata==2020.1 +PyYAML==5.4.1 +regex==2020.11.13 +reno==4.0.0 +repoze.lru==0.7 +requests==2.23.0 +requests-oauthlib==1.3.1 +resolver==0.2.1 +responses==0.12.1 +retry==0.9.2 +rfc3986==2.0.0 +Routes==2.5.1 +rsa==4.9 +sasl==0.3.1 +selector==0.10.1 +setproctitle==1.3.2 +setuptools==45.2.0 +simplejson==3.19.1 +six==1.16.0 +smmap==5.0.0 +snakebite-py3==3.0.5 +snowballstemmer==2.2.0 +Sphinx==3.3.1 +sphinx-rtd-theme==0.5.0 +sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +SQLAlchemy==1.3.20 +SQLAlchemy-JSONField==0.9.0 +sqlalchemy-migrate==0.13.0 +SQLAlchemy-Utils==0.41.0 +sqlparse==0.4.4 +statsd==4.0.1 +stevedore==5.0.0 +tabulate==0.8.10 +Tempita==0.5.2 +tenacity==4.12.0 +testfixtures==7.1.0 +testrepository==0.0.20 +testresources==2.0.1 +testscenarios==0.5.0 +testtools==2.5.0 +text-unidecode==1.3 +thrift==0.16.0 +thrift-sasl==0.4.3 +tiddlyweb==2.4.3 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.11.7 +tornado==5.1.1 +tox==3.12.1 +typing-extensions==3.7.2 +tzlocal==1.5.1 +ulid==1.1 +unicodecsv==0.14.1 +urllib3==1.25.11 +uWSGI==2.0.21 +vine==1.3.0 +virtualenv==20.22.0 +wcwidth==0.2.6 +WebOb==1.8.7 +websocket-client==1.5.1 +Werkzeug==0.16.1 +wheel==0.40.0 +wrapt==1.15.0 +wsgi-intercept==1.11.0 +WTForms==2.3.3 +xmltodict==0.13.0 +yappi==1.4.0 +yq==3.2.1 +zipp==3.15.0 +zope.deprecation==4.4.0 diff --git a/src/bin/shipyard_airflow/requirements.txt b/src/bin/shipyard_airflow/requirements.txt index a74ef24d..5d4c01fe 100644 --- a/src/bin/shipyard_airflow/requirements.txt +++ b/src/bin/shipyard_airflow/requirements.txt @@ -1,61 +1,3 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# API requirements - -alembic==1.0.1 -arrow==0.12.1 -celery==4.3 -kombu<4.7,>=4.6.10 -vine<5.0.0a1,>=1.1.3 -configparser==3.5.0 -cryptography>=2.7 -falcon==1.4.1 -jsonschema>=3.0.1,<4 -grpcio>=1.16.0 -keystoneauth1>=3.18.0 -keystonemiddleware==5.3.0 -networkx==2.2 # common/deployment_group -oslo.config==7.0.0 -oslo.policy==1.40.1 -PasteDeploy==1.5.2 -psycopg2-binary==2.8.4 -pylibyaml~=0.1 -oslo.cache==1.38.1 -oslo.log==3.45.2 -oslo.utils==3.42.1 -oslo.serialization==2.29.2 -pbr==5.4.5 -python-dateutil==2.8.1 -python-memcached==1.59 -python-keystoneclient==3.22.0 -requests!=2.20.0,>=2.14.2 -setuptools==40.4.3 -SQLAlchemy==1.3.15 -ulid==1.1 -urllib3==1.25.9 -uwsgi~=2.0.19.1 - -# To support profiling in non-prod -Werkzeug==0.16.1 -WTForms~=2.2.1 -# Need to lock marshmellow-sqlalchemy and tabulate for compatibility issues -marshmallow-sqlalchemy==0.18.0 -tabulate==0.8.03 - -# Dependencies for other UCP components -git+https://opendev.org/airship/deckhand.git@5cd799cc5d04527ac782270008ff647b3779ff05#egg=deckhand -git+https://opendev.org/airship/drydock.git@b431f7a2cfacd199fa5587487f372dc5589f9126#egg=drydock_provisioner&subdirectory=python -git+https://opendev.org/airship/armada.git@379c88d6195eed1a0ef61d0465d5746bcfda1a42#egg=armada -git+https://opendev.org/airship/promenade.git@c10165c144e0a18137596a3c89f1339d6ed30d0c#egg=promenade +# Warning: This file should be empty. +# Specify direct dependencies in requirements-direct.txt instead. +-r requirements-direct.txt diff --git a/src/bin/shipyard_airflow/setup.cfg b/src/bin/shipyard_airflow/setup.cfg index bd32ea03..a934df2e 100644 --- a/src/bin/shipyard_airflow/setup.cfg +++ b/src/bin/shipyard_airflow/setup.cfg @@ -12,8 +12,8 @@ classifier = Operating System :: POSIX :: Linux Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.10 [files] packages = diff --git a/src/bin/shipyard_airflow/shipyard_airflow/common/document_validators/document_validator.py b/src/bin/shipyard_airflow/shipyard_airflow/common/document_validators/document_validator.py index d0a00b37..027058a9 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/common/document_validators/document_validator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/common/document_validators/document_validator.py @@ -117,11 +117,11 @@ class DocumentValidator(metaclass=abc.ABCMeta): Triggers the specific checks after any common checks """ if self.missing_severity not in ["Error", "Warning", "Info"]: - LOG.warn("Document Validator for {}, {} does not have a valid " - "value set for missing_severity (got {}). " - "Assuming Error".format( - self.schema, self.doc_name, self.missing_severity - )) + LOG.warning("Document Validator for {}, {} does not have a valid " + "value set for missing_severity (got {}). " + "Assuming Error".format( + self.schema, self.doc_name, self.missing_severity + )) self.missing_severity = "Error" try: diff --git a/src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py index b8c081a2..4d22202e 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py @@ -147,7 +147,7 @@ class NotesHelper: note_timestamp=note_timestamp ) except Exception as ex: - LOG.warn( + LOG.warning( "Creating note for {} encountered a problem, exception info " "follows, but processing is not halted for notes.", assoc_id @@ -163,7 +163,7 @@ class NotesHelper: q = Query(assoc_id_pattern, verbosity, exact_match) return self.nm.retrieve(q) except Exception as ex: - LOG.warn( + LOG.warning( "Note retrieval for {} encountered a problem, exception " "info follows, but processing is not halted for notes.", assoc_id_pattern diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py index 7a132c5d..39d5bf1a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py @@ -109,7 +109,7 @@ class ActionsResource(BaseResource): Return actions that have been invoked through shipyard. :returns: a json array of action entities """ - resp.body = self.to_json(self.get_all_actions( + resp.text = self.to_json(self.get_all_actions( verbosity=req.context.verbosity) ) resp.status = falcon.HTTP_200 @@ -133,7 +133,7 @@ class ActionsResource(BaseResource): LOG.info("Id %s generated for action %s", action['id'], action['name']) # respond with the action and location for checking status resp.status = falcon.HTTP_201 - resp.body = self.to_json(action) + resp.text = self.to_json(action) resp.location = '/api/v1.0/actions/{}'.format(action['id']) def create_action(self, action, context, allow_intermediate_commits=False): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_id_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_id_api.py index f03d9ca3..edfa4558 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_id_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_id_api.py @@ -36,7 +36,7 @@ class ActionsIdResource(BaseResource): Return actions that have been invoked through shipyard. :returns: a json array of action entities """ - resp.body = self.to_json(self.get_action( + resp.text = self.to_json(self.get_action( action_id=kwargs['action_id'], verbosity=req.context.verbosity )) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_steps_id_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_steps_id_api.py index 60535b75..7d32237a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_steps_id_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_steps_id_api.py @@ -32,7 +32,7 @@ class ActionsStepsResource(BaseResource): Return step details for an action step :returns: a json object describing a step """ - resp.body = self.to_json( + resp.text = self.to_json( self.get_action_step( action_id=kwargs['action_id'], step_id=kwargs['step_id'], diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_steps_id_logs_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_steps_id_logs_api.py index 7943f76b..de76a54a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_steps_id_logs_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_steps_id_logs_api.py @@ -51,7 +51,7 @@ class ActionsStepsLogsResource(BaseResource): step_id = ActionsHelper.parse_step_id(**kwargs) # Retrieve logs for the action step - resp.body = self.get_action_step_logs(action_id, + resp.text = self.get_action_step_logs(action_id, step_id, try_number) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_validations_id_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_validations_id_api.py index 6e1f4da3..5fde3488 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_validations_id_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_validations_id_api.py @@ -31,7 +31,7 @@ class ActionsValidationsResource(BaseResource): Return validation details for an action validation :returns: a json object describing a validation """ - resp.body = self.to_json( + resp.text = self.to_json( self.get_action_validation(kwargs['action_id'], kwargs['validation_id'])) resp.status = falcon.HTTP_200 diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/af_monitoring/workflows_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/af_monitoring/workflows_api.py index 6f69957c..d56b1020 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/af_monitoring/workflows_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/af_monitoring/workflows_api.py @@ -38,7 +38,7 @@ class WorkflowResource(BaseResource): """ since_date = req.params.get('since') helper = WorkflowHelper(req.context.external_marker) - resp.body = self.to_json( + resp.text = self.to_json( self.get_all_workflows(helper=helper, since_date=since_date) ) resp.status = falcon.HTTP_200 @@ -67,7 +67,7 @@ class WorkflowIdResource(BaseResource): :returns: a json object of a workflow entity """ helper = WorkflowHelper(req.context.external_marker) - resp.body = self.to_json( + resp.text = self.to_json( self.get_workflow_detail(helper=helper, workflow_id=workflow_id) ) resp.status = falcon.HTTP_200 diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/api.py index c92af916..9a730af5 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/api.py @@ -57,7 +57,7 @@ def start_api(): LoggingMiddleware(), CommonParametersMiddleware() ] - control_api = falcon.API( + control_api = falcon.App( request_type=ShipyardRequest, middleware=middlewares) control_api.add_route('/versions', VersionsResource()) @@ -113,7 +113,7 @@ class VersionsResource(BaseResource): """ def on_get(self, req, resp): - resp.body = self.to_json({ + resp.text = self.to_json({ 'v1.0': { 'path': '/api/v1.0', 'status': 'stable' diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/base.py b/src/bin/shipyard_airflow/shipyard_airflow/control/base.py index 21e23865..2bde201a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/base.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/base.py @@ -33,7 +33,7 @@ class BaseResource(object): """ def on_options(self, req, resp, **kwargs): """Handle options requests""" - method_map = routing.create_http_method_map(self) + method_map = routing.map_http_methods(self) for method in method_map: if method_map.get(method).__name__ != 'method_not_allowed': resp.append_header('Allow', method) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py index 998aa787..1c6c6664 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py @@ -48,7 +48,7 @@ class ConfigDocsStatusResource(BaseResource): """Returns a list of the configdocs and their statuses""" versions = req.params.get('versions') or None helper = ConfigdocsHelper(req.context) - resp.body = self.to_json(helper.get_configdocs_status(versions)) + resp.text = self.to_json(helper.get_configdocs_status(versions)) resp.status = falcon.HTTP_200 @@ -90,7 +90,7 @@ class ConfigDocsResource(BaseResource): if validations and validations['status'] == 'Success': validations['code'] = resp.status resp.location = '/api/v1.0/configdocs/{}'.format(collection_id) - resp.body = self.to_json(validations) + resp.text = self.to_json(validations) def validate_content_length(self, content_length): """Validates that the content length header is valid @@ -131,7 +131,7 @@ class ConfigDocsResource(BaseResource): policy.check_auth(req.context, policy.GET_CONFIGDOCS_CLRTXT) # Not reformatting to JSON or YAML since just passing through - resp.body = self.get_collection( + resp.text = self.get_collection( helper=helper, collection_id=collection_id, version=version, cleartext_secrets=cleartext_secrets) resp.append_header('Content-Type', 'application/x-yaml') @@ -277,7 +277,7 @@ class CommitConfigDocsResource(BaseResource): dryrun = req.get_param_as_bool(name='dryrun') or False helper = ConfigdocsHelper(req.context) validations = self.commit_configdocs(helper, force, dryrun) - resp.body = self.to_json(validations) + resp.text = self.to_json(validations) resp.status = validations.get('code', falcon.HTTP_200) def commit_configdocs(self, helper, force, dryrun): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/rendered_configdocs_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/rendered_configdocs_api.py index e5c24949..c6ba33d1 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/rendered_configdocs_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/rendered_configdocs_api.py @@ -52,7 +52,7 @@ class RenderedConfigDocsResource(BaseResource): policy.check_auth(req.context, policy.GET_RENDEREDCONFIGDOCS_CLRTXT) - resp.body = self.get_rendered_configdocs( + resp.text = self.get_rendered_configdocs( helper=helper, version=version, cleartext_secrets=cleartext_secrets diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/middleware/common_params.py b/src/bin/shipyard_airflow/shipyard_airflow/control/middleware/common_params.py index 86d5fce5..d315c4f4 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/middleware/common_params.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/middleware/common_params.py @@ -49,7 +49,10 @@ class CommonParametersMiddleware(object): try: verbosity = req.get_param_as_int( - 'verbosity', required=False, min=0, max=MAX_VERBOSITY + 'verbosity', + required=False, + min_value=0, + max_value=MAX_VERBOSITY ) if verbosity is not None: # if not set, retains the context default value. diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/middleware/logging_mw.py b/src/bin/shipyard_airflow/shipyard_airflow/control/middleware/logging_mw.py index ae26be40..7b234b80 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/middleware/logging_mw.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/middleware/logging_mw.py @@ -60,7 +60,7 @@ class LoggingMiddleware(object): # have response scrubbing or way to categorize responses in the # future, this may be an appropriate place to utilize it. if resp_code >= 400: - LOG.debug('Errored Response body: %s', resp.body) + LOG.debug('Errored Response body: %s', resp.text) def _log_headers(self, headers): """ Log request headers, while scrubbing sensitive values diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/notes/notedetails_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/notes/notedetails_api.py index ece1ce80..68c5baf2 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/notes/notedetails_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/notes/notedetails_api.py @@ -47,7 +47,7 @@ class NoteDetailsResource(BaseResource): note_id = kwargs['note_id'] self.validate_note_id(note_id) note = self.get_note_with_access_check(req.context, note_id) - resp.body = self.get_note_details(note) + resp.text = self.get_note_details(note) resp.status = falcon.HTTP_200 def validate_note_id(self, note_id): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/status/status_api.py b/src/bin/shipyard_airflow/shipyard_airflow/control/status/status_api.py index 31629fa5..4205844a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/status/status_api.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/status/status_api.py @@ -42,5 +42,5 @@ class StatusResource(BaseResource): else: fltrs = None helper = StatusHelper(req.context) - resp.body = self.to_json(helper.get_site_statuses(fltrs)) + resp.text = self.to_json(helper.get_site_statuses(fltrs)) resp.status = falcon.HTTP_200 diff --git a/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_action.py b/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_action.py index 826c0032..91b087a3 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_action.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_action.py @@ -57,8 +57,9 @@ class ValidateDeploymentAction: results = self.doc_val_mgr.validate() if self.doc_val_mgr.errored: if self.cont_on_fail: - LOG.warn("Validation failures occured, but 'continue-on-fail' " - "is set to true. Processing continues") + LOG.warning("Validation failures occured, " + "but 'continue-on-fail' " + "is set to true. Processing continues") else: raise ApiError( title='Document validation failed', diff --git a/src/bin/shipyard_airflow/shipyard_airflow/errors.py b/src/bin/shipyard_airflow/shipyard_airflow/errors.py index 906c9eac..bf4cdec7 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/errors.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/errors.py @@ -104,7 +104,7 @@ def format_error_resp(req, 'retry': retry } - resp.body = json.dumps(error_response, default=str) + resp.text = json.dumps(error_response, default=str) resp.content_type = 'application/json' resp.status = status_code diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py index 47d5b615..4e8d2d69 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py @@ -404,10 +404,11 @@ class DrydockBaseOperator(UcpBaseOperator): task_status = task_dict.get('status', "Not Specified") task_result = task_dict.get('result') if task_result is None: - LOG.warn("Task result is missing for task %s, with status %s." - " Neither successes nor further details can be" - " extracted from this result", - task_id, task_status) + LOG.warning("Task result is missing for task %s, " + "with status %s." + " Neither successes nor further details can be" + " extracted from this result", + task_id, task_status) else: if extend_success: try: @@ -429,7 +430,7 @@ class DrydockBaseOperator(UcpBaseOperator): ", ".join(success_nodes), task_id) except KeyError: # missing key on the path to getting nodes - don't add - LOG.warn( + LOG.warning( "Missing successes field on result of task %s, " "but a success field was expected. No successes" " can be extracted from this result", task_id @@ -448,8 +449,8 @@ class DrydockBaseOperator(UcpBaseOperator): except Exception: # since we are reporting task results, if we can't get the # results, do not block the processing. - LOG.warn("Failed to retrieve a result for task %s. Exception " - "follows:", task_id, exc_info=True) + LOG.warning("Failed to retrieve a result for task %s. Exception " + "follows:", task_id, exc_info=True) # deduplicate and return return set(success_nodes) @@ -507,8 +508,8 @@ class DrydockBaseOperator(UcpBaseOperator): note_timestamp=msg.get('ts'), verbosity=3) except Exception as ex: - LOG.warn("Error while creating a task result note, " - "processing continues. Source info %s", msg) + LOG.warning("Error while creating a task result note, " + "processing continues. Source info %s", msg) LOG.exception(ex) links = task_result.get('links', []) @@ -528,8 +529,8 @@ class DrydockBaseOperator(UcpBaseOperator): is_auth_link=True, verbosity=5) except Exception as ex: - LOG.warn("Error while creating a link-based note, " - "processing continues. Source info: %s", link) + LOG.warning("Error while creating a link-based note, " + "processing continues. Source info: %s", link) LOG.exception(ex) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_destroy_nodes.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_destroy_nodes.py index 35d5d801..b5b646c4 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_destroy_nodes.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_destroy_nodes.py @@ -78,10 +78,11 @@ class DrydockDestroyNodeOperator(DrydockBaseOperator): "destroyed. The report at the end of processing " "this step contains the results", task_name) except DrydockTaskTimeoutException: - LOG.warn("Task %s has timed out after %s seconds. Some nodes may " - "have been destroyed. The report at the end of " - "processing this step contains the results", task_name, - self.dest_timeout) + LOG.warning("Task %s has timed out after %s seconds. " + "Some nodes may " + "have been destroyed. The report at the end of " + "processing this step contains the results", task_name, + self.dest_timeout) def report_summary(self): """Reports the successfully destroyed nodes""" diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py index 3bb65feb..5202c1bd 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py @@ -252,7 +252,7 @@ class DrydockNodesOperator(DrydockBaseOperator): except DrydockTaskFailedException: # Task failure may be successful enough based on success criteria. # This should not halt the overall flow of this workflow step. - LOG.warn( + LOG.warning( "Task %s with Drydock task-id: %s has failed. Logs contain " "details of the failure. Some nodes may be succesful, " "processing continues", task_name, self.drydock_task_id @@ -263,7 +263,7 @@ class DrydockNodesOperator(DrydockBaseOperator): except DrydockTaskTimeoutException: # Task timeout may be successful enough based on success criteria. # This should not halt the overall flow of this workflow step. - LOG.warn( + LOG.warning( "Task %s with Drydock task-id: %s has timed out after %s " "seconds. Logs contain details of the failure. Some nodes may " "be succesful, processing continues", task_name, diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_relabel_nodes.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_relabel_nodes.py index 531f5ac4..376b6b1c 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_relabel_nodes.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_relabel_nodes.py @@ -81,10 +81,11 @@ class DrydockRelabelNodesOperator(DrydockBaseOperator): "relabeled. The report at the end of processing " "this step contains the results", task_name) except DrydockTaskTimeoutException: - LOG.warn("Task %s has timed out after %s seconds. Some nodes may " - "have been relabeled. The report at the end of " - "processing this step contains the results", task_name, - self.task_timeout) + LOG.warning("Task %s has timed out after %s seconds. " + "Some nodes may " + "have been relabeled. The report at the end of " + "processing this step contains the results", task_name, + self.task_timeout) def report_summary(self): """Reports the successfully relabeled nodes""" diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py index a47c680a..77f4a56e 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py @@ -66,7 +66,8 @@ class DrydockValidateDesignOperator(DrydockBaseOperator): try: design_validate_response = requests.post(validation_endpoint, headers=headers, - data=json.dumps(payload)) + data=json.dumps(payload), + timeout=5) except requests.exceptions.RequestException as e: raise AirflowException(e) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py index aef8148b..213b3573 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py @@ -66,7 +66,8 @@ class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator): try: design_validate_response = requests.post(validation_endpoint, headers=headers, - data=json.dumps(payload)) + data=json.dumps(payload), + timeout=5) except requests.exceptions.RequestException as e: raise AirflowException(e) diff --git a/src/bin/shipyard_airflow/test-requirements.txt b/src/bin/shipyard_airflow/test-requirements.txt index 726a8b2e..d1800ecb 100644 --- a/src/bin/shipyard_airflow/test-requirements.txt +++ b/src/bin/shipyard_airflow/test-requirements.txt @@ -1,21 +1,8 @@ -# Testing -pyflakes>=2.1.1 -amqp~=2.6.0 +bandit==1.6.0 +flake8==3.8.4 pytest==3.5.0 pytest-cov==2.5.1 -responses==0.10.2 -testfixtures==5.1.1 +testfixtures +tox<=4.0.0 typing-extensions==3.7.2 -# Pin apache-airflow 1.10.5 to align with version in airflow images requirements.txt -apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5 -# install postgres seperately as apache-airflow 1.10.5 forces postgres to use -# psycopg2 instead of psycopg2-binary which requires additional apt packages -# to be installed, i.e. postgres-devel. Otherwise the following error is seen: -# Error: pg_config executable not found. Only seems to be an issue in xenial -postgres>=3.0.0 - -# Linting -flake8<3.8.0,>=3.6.0 - -# Security scanning -bandit>=1.5.0 # Apache-2.0 +pyflakes>=2.1.1 \ No newline at end of file diff --git a/src/bin/shipyard_airflow/tests/unit/control/common.py b/src/bin/shipyard_airflow/tests/unit/control/common.py index e49a5eda..b8b7fde0 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/common.py +++ b/src/bin/shipyard_airflow/tests/unit/control/common.py @@ -39,12 +39,12 @@ def create_req(ctx, body): env = testing.create_environ( path='/', query_string='', - protocol='HTTP/1.1', + http_version='1.1', scheme='http', host='falconframework.org', port=None, headers={'Content-Type': 'application/json'}, - app='', + root_path='', body=body, method='POST', wsgierrors=None, diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py b/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py index 575b3137..86b2d85b 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py @@ -80,12 +80,12 @@ def create_req(ctx, body): env = testing.create_environ( path='/', query_string='', - protocol='HTTP/1.1', + http_version='1.1', scheme='http', host='falconframework.org', port=None, headers={'Content-Type': 'application/json'}, - app='', + root_path='', body=body, method='POST', wsgierrors=None, @@ -284,7 +284,7 @@ def test_on_get(mock_get_all_actions, mock_authorize): mock_authorize.assert_called_once_with( 'workflow_orchestrator:list_actions', context) assert mock_get_all_actions.call_count == 1 - assert resp.body is not None + assert resp.text is not None assert resp.status == '200 OK' @@ -318,7 +318,7 @@ def test_on_post(mock_info, mock_create_action, mock_authorize, *args): mock_info.assert_called_with("Id %s generated for action %s", 'test_id', 'test_name') assert resp.status == '201 Created' - assert resp.body is not None + assert resp.text is not None assert '/api/v1.0/actions/' in resp.location diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_actions_id_api.py b/src/bin/shipyard_airflow/tests/unit/control/test_actions_id_api.py index af7a0a1d..5288ff18 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_actions_id_api.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_actions_id_api.py @@ -166,7 +166,7 @@ def test_on_get(mock_authorize, mock_get_action): mock_authorize.assert_called_once_with('workflow_orchestrator:get_action', context) mock_get_action.assert_called_once_with(action_id=None, verbosity=1) - assert resp.body == '"action_returned"' + assert resp.text == '"action_returned"' assert resp.status == '200 OK' @mock.patch('shipyard_airflow.control.helpers.action_helper.notes_helper', diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_actions_steps_id_logs_api.py b/src/bin/shipyard_airflow/tests/unit/control/test_actions_steps_id_logs_api.py index 2b69629d..1244192e 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_actions_steps_id_logs_api.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_actions_steps_id_logs_api.py @@ -121,7 +121,7 @@ Starting attempt 1 of 1 -------------------------------------------------------------------------------- Executing Task(PythonOperator): action_xcom -Running: ['bash', '-c', 'airflow run deploy_site action_xcom \ +Running: ['bash', '-c', 'airflow tasks run deploy_site action_xcom \ 2018-04-11T07:30:37 --job_id 2 --raw -sd DAGS_FOLDER/deploy_site.py'] Running on host airflow-worker-0.airflow-worker-discovery.ucp.svc.cluster.local Subtask: [2018-04-11 07:30:43,944] {{python_operator.py:90}} \ diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_api.py b/src/bin/shipyard_airflow/tests/unit/control/test_api.py index 971ffc37..28e08218 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_api.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_api.py @@ -24,6 +24,6 @@ class TestVersionsResource(): req = create_req(context, None) resp = create_resp() version_resource.on_get(req, resp) - assert sorted(resp.body) == sorted( + assert sorted(resp.text) == sorted( '{"v1.0": {"status": "stable", "path": "/api/v1.0"}}') assert resp.status == '200 OK' diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py b/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py index 519dc150..9e043d56 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py @@ -46,7 +46,7 @@ class TestConfigDocsStatusResource(): assert result.status_code == 200 assert result.text == json.dumps(common.str_responder(), default=str) assert result.headers[ - 'content-type'] == 'application/json; charset=UTF-8' + 'content-type'] == 'application/json' class TestConfigDocsResource(): diff --git a/src/bin/shipyard_airflow/tests/unit/control/test_status_api.py b/src/bin/shipyard_airflow/tests/unit/control/test_status_api.py index cea044bb..e4608168 100644 --- a/src/bin/shipyard_airflow/tests/unit/control/test_status_api.py +++ b/src/bin/shipyard_airflow/tests/unit/control/test_status_api.py @@ -33,4 +33,4 @@ class TestStatusResource(): assert result.status_code == 200 assert result.text == json.dumps(common.str_responder(), default=str) assert result.headers[ - 'content-type'] == 'application/json; charset=UTF-8' + 'content-type'] == 'application/json' diff --git a/src/bin/shipyard_airflow/tox.ini b/src/bin/shipyard_airflow/tox.ini index 5a8272b7..f29c1e6f 100644 --- a/src/bin/shipyard_airflow/tox.ini +++ b/src/bin/shipyard_airflow/tox.ini @@ -1,15 +1,29 @@ [tox] skipsdist=True -envlist = py36, pep8, bandit +envlist = py38, pep8, bandit [testenv] setenv= PYTHONWARNING=all LANG=C.UTF-8 LC_ALL=C.UTF-8 -deps = -r{toxinidir}/requirements.txt +deps = -r{toxinidir}/requirements-frozen.txt -[testenv:py36] + +[testenv:freeze] +recreate = True +allowlist_externals= + rm + sh +deps= + -r{toxinidir}/requirements-direct.txt + -r{toxinidir}/test-requirements.txt +commands= + rm -f {toxinidir}/requirements-frozen.txt + sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt" + + +[testenv:py38] skipsdist=True setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes @@ -22,10 +36,10 @@ commands = --cov-report=term-missing:skip-covered \ --cov-config=.coveragerc \ --cov=shipyard_airflow \ - --cov-report=html + --cov-report=html \ + -vv [testenv:pep8] -basepython=python3 deps= flake8>=3.3.0 bandit>=1.5.0 @@ -34,7 +48,6 @@ commands = bandit -r shipyard_airflow [testenv:bandit] -basepython=python3 skipsdist= True commands = bandit -r shipyard_airflow -n 5 @@ -42,20 +55,32 @@ commands = [testenv:genconfig] setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes -commands = oslo-config-generator --config-file=generator/config-generator.conf +deps = + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements-frozen.txt +commands = + pip install . --use-pep517 + oslo-config-generator --config-file=generator/config-generator.conf [testenv:genpolicy] setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes -commands = oslopolicy-sample-generator --config-file=generator/policy-generator.conf +deps = + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements-frozen.txt +commands = + pip install . --use-pep517 + oslopolicy-sample-generator --config-file=generator/policy-generator.conf [testenv:gen_all] # Combined to make these run together instead of setting up separately -basepython=python3 setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes +deps = + -r{toxinidir}/test-requirements.txt + -r{toxinidir}/requirements-frozen.txt commands = - pip install -r{toxinidir}/test-requirements.txt + pip install . --use-pep517 oslo-config-generator --config-file=generator/config-generator.conf oslopolicy-sample-generator --config-file=generator/policy-generator.conf diff --git a/src/bin/shipyard_client/requirements-direct.txt b/src/bin/shipyard_client/requirements-direct.txt new file mode 100644 index 00000000..e663fa66 --- /dev/null +++ b/src/bin/shipyard_client/requirements-direct.txt @@ -0,0 +1,24 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# CLI/Client requirements + +arrow<=0.17.0 +click==6.7 +click-default-group==1.2 +keystoneauth1<=5.1.1 +requests==2.23.0 +PTable==0.9.2 +pylibyaml==0.1.0 +PyYAML<=5.4.1 diff --git a/src/bin/shipyard_client/requirements-frozen.txt b/src/bin/shipyard_client/requirements-frozen.txt new file mode 100644 index 00000000..a88e5c46 --- /dev/null +++ b/src/bin/shipyard_client/requirements-frozen.txt @@ -0,0 +1,40 @@ +amqp==2.6.1 +arrow==0.17.0 +attrs==23.1.0 +bandit==1.6.0 +certifi==2022.12.7 +chardet==3.0.4 +click==6.7 +click-default-group==1.2 +coverage==7.2.3 +flake8==3.8.4 +gitdb==4.0.10 +GitPython==3.1.31 +idna==2.10 +iso8601==1.1.0 +keystoneauth1==5.1.1 +mccabe==0.6.1 +more-itertools==9.1.0 +os-service-types==1.7.0 +pbr==5.11.1 +pip==23.0.1 +pluggy==0.6.0 +PTable==0.9.2 +py==1.11.0 +pycodestyle==2.6.0 +pyflakes==2.2.0 +pylibyaml==0.1.0 +pytest==3.5.0 +pytest-cov==2.5.1 +python-dateutil==2.8.2 +PyYAML==5.4.1 +requests==2.23.0 +responses==0.10.2 +setuptools==67.6.0 +six==1.16.0 +smmap==5.0.0 +stevedore==5.0.0 +testfixtures==5.1.1 +urllib3==1.25.11 +vine==1.3.0 +wheel==0.40.0 diff --git a/src/bin/shipyard_client/requirements.txt b/src/bin/shipyard_client/requirements.txt index cf2e32ce..5d4c01fe 100644 --- a/src/bin/shipyard_client/requirements.txt +++ b/src/bin/shipyard_client/requirements.txt @@ -1,24 +1,3 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# CLI/Client requirements - -arrow==0.12.1 -click==6.7 -click-default-group==1.2 -keystoneauth1>=3.18.0 -requests!=2.20.0,>=2.14.2 -PTable==0.9.2 -pylibyaml==0.1.0 -pyyaml~=5.1 +# Warning: This file should be empty. +# Specify direct dependencies in requirements-direct.txt instead. +-r requirements-direct.txt diff --git a/src/bin/shipyard_client/setup.cfg b/src/bin/shipyard_client/setup.cfg index 25294768..5192c2eb 100644 --- a/src/bin/shipyard_client/setup.cfg +++ b/src/bin/shipyard_client/setup.cfg @@ -12,8 +12,8 @@ classifier = Operating System :: POSIX :: Linux Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.10 [files] packages = diff --git a/src/bin/shipyard_client/test-requirements.txt b/src/bin/shipyard_client/test-requirements.txt index 515ab1ae..07678d40 100644 --- a/src/bin/shipyard_client/test-requirements.txt +++ b/src/bin/shipyard_client/test-requirements.txt @@ -1,14 +1,14 @@ # Testing -pyflakes>=2.1.1 -amqp~=2.6.0 +pyflakes==2.2.0 +amqp==2.6.1 pytest==3.5.0 pytest-cov==2.5.1 responses==0.10.2 testfixtures==5.1.1 # Linting -flake8<3.8.0,>=3.6.0 +flake8==3.8.4 # Security scanning -bandit>=1.1.0 # Apache-2.0 +bandit==1.6.0 # Apache-2.0 diff --git a/src/bin/shipyard_client/tox.ini b/src/bin/shipyard_client/tox.ini index 12464ead..703a9ea1 100644 --- a/src/bin/shipyard_client/tox.ini +++ b/src/bin/shipyard_client/tox.ini @@ -1,16 +1,29 @@ [tox] skipsdist=True -envlist = py36, pep8, bandit +envlist = py38, pep8, bandit [testenv] setenv= PYTHONWARNING=all LC_ALL=C.UTF-8 LC_ALL=C.UTF-8 -deps = -r{toxinidir}/requirements.txt +deps = -r{toxinidir}/requirements-frozen.txt -r{toxinidir}/test-requirements.txt -[testenv:py36] +[testenv:freeze] +recreate = True +allowlist_externals= + rm + sh +deps= + -r{toxinidir}/requirements-direct.txt + -r{toxinidir}/test-requirements.txt +commands= + rm -f {toxinidir}/requirements-frozen.txt + sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt" + + +[testenv:py38] skipsdist=True commands = pytest \ @@ -19,10 +32,10 @@ commands = --cov-report=term-missing:skip-covered \ --cov-config=.coveragerc \ --cov=shipyard_client \ - --cov-report=html + --cov-report=html \ + -vv [testenv:pep8] -basepython=python3 deps= flake8>=3.3.0 bandit>=1.5.0 @@ -31,7 +44,6 @@ commands = bandit -r shipyard_airflow [testenv:bandit] -basepython=python3 skipsdist=True commands = bandit -r shipyard_client -n 5 diff --git a/tools/airflow_image_run.sh b/tools/airflow_image_run.sh index dd5d2fcb..37fe5a3b 100755 --- a/tools/airflow_image_run.sh +++ b/tools/airflow_image_run.sh @@ -47,7 +47,7 @@ fi docker stop airflow_test docker rm airflow_test -if [[ ${TEST_RESP:(-7)} == "success" ]]; then +if [ ${TEST_RESP:(-7)} == "success" ]; then exit 0 else exit 1 diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index 05687aff..fc589bf3 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -1,4 +1,4 @@ -# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# Copyright 2019, AT&T Intellectual Property # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,24 +13,15 @@ # limitations under the License. - hosts: primary + roles: + - clear-firewall + - bindep + - disable-systemd-resolved + - ensure-python + - ensure-pip + - ensure-docker + tasks: - # Stop systemd-resolved service before starting docker. - - name: stop systemd-resolved service - systemd: - state: stopped - enabled: no - masked: yes - daemon_reload: yes - name: systemd-resolved - become: yes - - - name: ensure pip3 installed - apt: - name: "{{ item }}" - with_items: - - python3-pip - become: yes - - name: Clone Required Repositories shell: | export CLONE_SHIPYARD={{ CLONE_SHIPYARD }} @@ -39,7 +30,7 @@ args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" - - name: Setup AppArmor + - name: Setup Apparmor shell: | ./tools/deployment/airskiff/developer/009-setup-apparmor.sh args: @@ -58,16 +49,22 @@ chdir: "{{ zuul.project.src_dir }}" become: yes + - name: Setup clients + shell: | + ./tools/deployment/airskiff/developer/020-setup-client.sh + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + - name: Deploy Airship components using Armada shell: | - mkdir ~/.kube + mkdir -p ~/.kube cp -rp /home/zuul/.kube/config ~/.kube/config ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" become: yes - - name: Deploy OpenStack using Airship + - name: Deploy Memcached using Airship shell: | ./tools/deployment/airskiff/developer/100-deploy-osh.sh args: diff --git a/tools/gate/playbooks/airskiff-reduce-site.yaml b/tools/gate/playbooks/airskiff-reduce-site.yaml index b57dc3a6..7c80ed77 100644 --- a/tools/gate/playbooks/airskiff-reduce-site.yaml +++ b/tools/gate/playbooks/airskiff-reduce-site.yaml @@ -14,40 +14,10 @@ - hosts: primary tasks: - - name: ensure pip packages are installed - apt: - name: "{{ item }}" - with_items: - - python3-pip - - python3-git - - python3-requests - - python3-yaml - become: yes - - - - name: deploy iptables packages - include_role: - name: deploy-package - tasks_from: dist - vars: - packages: - deb: - - iptables - rpm: - - iptables - - command: sudo iptables -S - - command: sudo iptables -F - - command: sudo iptables -P INPUT ACCEPT - - command: sudo iptables -S - become: yes - - name: Overwrite Armada manifest shell: | git checkout v1.9 mv tools/gate/manifests/full-site.yaml \ type/skiff/manifests/full-site.yaml - # Overwrite versions.yaml with the latest Airship charts and xenial images - ./tools/updater.py --in-file global/software/config/versions.yaml \ - --tag-filter ubuntu_xenial --skip mariadb args: chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" diff --git a/tools/gate/playbooks/run-image.yaml b/tools/gate/playbooks/run-image.yaml index 3514bdee..80e025f8 100644 --- a/tools/gate/playbooks/run-image.yaml +++ b/tools/gate/playbooks/run-image.yaml @@ -18,6 +18,16 @@ vars_files: - vars.yaml roles: + - bindep + - clear-firewall + - ensure-python + - ensure-pip + - ensure-docker - build-images tags: + - bindep + - clear-firewall + - ensure-python + - ensure-pip + - ensure-docker - build-images diff --git a/tools/gate/roles/build-images/files/docker-systemd.conf b/tools/gate/roles/build-images/files/docker-systemd.conf deleted file mode 100644 index 6b01af0f..00000000 --- a/tools/gate/roles/build-images/files/docker-systemd.conf +++ /dev/null @@ -1,8 +0,0 @@ -# NOTE(SamYaple): CentOS cannot be build with userns-remap enabled. httpd uses -# cap_set_file capability and there is no way to pass that in at build as of -# docker 17.06. -# TODO(SamYaple): Periodically check to see if this is possible in newer -# versions of Docker -[Service] -ExecStart= -ExecStart=/usr/bin/dockerd diff --git a/tools/gate/roles/build-images/tasks/airship-shipyard.yaml b/tools/gate/roles/build-images/tasks/airship-shipyard.yaml index 100b44ab..63ec9353 100644 --- a/tools/gate/roles/build-images/tasks/airship-shipyard.yaml +++ b/tools/gate/roles/build-images/tasks/airship-shipyard.yaml @@ -38,52 +38,15 @@ debug: var: image_tags -- name: Install Docker (Debian) - when: ansible_os_family == 'Debian' +- name: Install Docker python module for ansible docker login block: - - file: - path: "{{ item }}" - state: directory - with_items: - - /etc/docker/ - - /etc/systemd/system/docker.service.d/ - - /var/lib/docker/ - - mount: - path: /var/lib/docker/ - src: tmpfs - fstype: tmpfs - opts: size=25g - state: mounted - - copy: "{{ item }}" - with_items: - - content: "{{ docker_daemon | to_json }}" - dest: /etc/docker/daemon.json - - src: files/docker-systemd.conf - dest: /etc/systemd/system/docker.service.d/ - - apt_key: - url: https://download.docker.com/linux/ubuntu/gpg - - apt_repository: - repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker/{{ ansible_distribution_release }} {{ ansible_distribution_release }} stable - - apt: - name: "{{ item }}" - allow_unauthenticated: True - with_items: - - docker-ce - - python3-pip - - python3-setuptools - pip: name: docker - version: 2.7.0 + version: 4.4.4 executable: pip3 - # NOTE(SamYaple): Allow all connections from containers to host so the - # containers can access the http server for git and wheels - - iptables: - action: insert - chain: INPUT - in_interface: docker0 - jump: ACCEPT become: True + - name: Make images when: not publish block: diff --git a/tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml b/tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml new file mode 100644 index 00000000..0eda1fbf --- /dev/null +++ b/tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml @@ -0,0 +1,37 @@ +# Copyright 2020 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Disable systemd-resolved service + systemd: + state: stopped + enabled: no + masked: yes + daemon_reload: yes + name: systemd-resolved + become: yes + +- name: Remove local stub dns from resolv.conf, if it exists + lineinfile: + path: /etc/resolv.conf + state: absent + regexp: '^nameserver.*127.0.0.1' + become: yes + +- name: Add upstream nameservers in resolv.conf + blockinfile: + path: /etc/resolv.conf + block: | + nameserver 8.8.8.8 + nameserver 8.8.4.4 + become: yes diff --git a/tools/gate/roles/disable-systemd-resolved/tasks/main.yaml b/tools/gate/roles/disable-systemd-resolved/tasks/main.yaml new file mode 100644 index 00000000..bb381b41 --- /dev/null +++ b/tools/gate/roles/disable-systemd-resolved/tasks/main.yaml @@ -0,0 +1,15 @@ +# Copyright 2020 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- include: disable-systemd-resolved.yaml diff --git a/tools/helm_install.sh b/tools/helm_install.sh index 730d1413..cee81412 100755 --- a/tools/helm_install.sh +++ b/tools/helm_install.sh @@ -17,7 +17,7 @@ set -x HELM=$1 -HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz"} +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz"} function install_helm_binary { diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index 9b496a22..b90d5198 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -17,7 +17,9 @@ set -eux HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"} -HTK_STABLE_COMMIT=${HTK_COMMIT:-"5c4056ad341afcc577e63902b6ddbfb222d757e1"} +HTK_STABLE_COMMIT=${HTK_COMMIT:-"a7cd689280cdbc0acd04a7a1b745941260e8700b"} + + TMP_DIR=$(mktemp -d) diff --git a/tox.ini b/tox.ini index a9cffbf9..a64b0859 100644 --- a/tox.ini +++ b/tox.ini @@ -5,15 +5,20 @@ envlist = all_jobs, docs [testenv] setenv= PYTHONWARNING=all -passenv=HTTP_PROXY HTTPS_PROXY http_proxy https_proxy NO_PROXY no_proxy +passenv= + HTTP_PROXY + HTTPS_PROXY + http_proxy + https_proxy + NO_PROXY + no_proxy [testenv:venv] commands = {posargs} # Please use `make docs` instead [testenv:docs] -basepython=python3 -whitelist_externals= +allowlist_externals= rm cp tox @@ -28,22 +33,21 @@ deps= -r{toxinidir}/doc/requirements.txt # `all_jobs` is here to support invocation as simply `tox`, as is done in # current pipeline. [testenv:all_jobs] -whitelist_externals= +allowlist_externals= tox commands= tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -[testenv:py36] -whitelist_externals= +[testenv:py38] +allowlist_externals= tox commands= - tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py36 - tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py36 + tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py38 + tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py38 [testenv:pep8] -basepython=python3 -whitelist_externals= +allowlist_externals= tox commands= tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e pep8