diff --git a/.zuul.yaml b/.zuul.yaml index aa331e00..e4bfb841 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -14,31 +14,29 @@ templates: - openstack-cover-jobs - openstack-python38-jobs - - openstack-python36-jobs + check: jobs: - openstack-tox-docs - openstack-tox-pep8 - armada-chart-build-gate - armada-chart-build-latest-htk - - armada-docker-build-gate-ubuntu_bionic - armada-docker-build-gate-ubuntu_focal - armada-airskiff-deploy-focal - - armada-airskiff-deploy-bionic + gate: jobs: - openstack-tox-docs - openstack-tox-pep8 - armada-chart-build-gate - - armada-docker-build-gate-ubuntu_bionic - armada-docker-build-gate-ubuntu_focal - armada-airskiff-deploy-focal - - armada-airskiff-deploy-bionic + post: jobs: - armada-docker-publish-ubuntu_focal - - armada-docker-publish-ubuntu_bionic - armada-upload-git-mirror + promote: jobs: - promote-airship-project-docs @@ -77,19 +75,6 @@ vars: HTK_COMMIT: master -- job: - name: armada-docker-build-gate-ubuntu_bionic - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node - vars: - publish: false - distro: ubuntu_bionic - tags: - dynamic: - patch_set: true - - - job: name: armada-docker-build-gate-ubuntu_focal timeout: 1800 @@ -118,7 +103,7 @@ - airship/treasuremap vars: CLONE_ARMADA: false - OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74 + OSH_INFRA_COMMIT: 3d8935a536faaa87e084e7783ed53f4660a3a1f8 ARMADA_IMAGE_DISTRO: ubuntu_focal irrelevant-files: - ^.*\.rst$ @@ -127,34 +112,6 @@ - ^releasenotes/.*$ - ^swagger/.*$ - -- job: - name: armada-airskiff-deploy-bionic - nodeset: armada-single-node-focal - description: | - Deploy Memcached using Airskiff and submitted Armada changes. - timeout: 9600 - voting: true - pre-run: - - tools/gate/playbooks/airskiff-reduce-site.yaml - - tools/gate/playbooks/git-config.yaml - run: tools/gate/playbooks/airskiff-deploy.yaml - post-run: tools/gate/playbooks/debug-report.yaml - required-projects: - - airship/treasuremap - vars: - CLONE_ARMADA: false - OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74 - ARMADA_IMAGE_DISTRO: ubuntu_bionic - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - ^examples/.*$ - - ^releasenotes/.*$ - - ^swagger/.*$ - - - - job: name: armada-docker-publish-ubuntu_focal timeout: 1800 @@ -172,22 +129,6 @@ static: - latest -- job: - name: armada-docker-publish-ubuntu_bionic - timeout: 1800 - run: tools/gate/playbooks/docker-image-build.yaml - nodeset: armada-single-node - secrets: - - airship_armada_quay_creds - vars: - publish: true - distro: ubuntu_bionic - tags: - dynamic: - branch: true - commit: true - static: - - latest - secret: name: airship_armada_quay_creds diff --git a/armada/handlers/metrics.py b/armada/handlers/metrics.py index e344f9ee..c943819a 100644 --- a/armada/handlers/metrics.py +++ b/armada/handlers/metrics.py @@ -115,13 +115,15 @@ class ActionWithTimeoutMetrics(ActionMetrics): timeout.observe(timeout_value) - def observe_timeout_usage(duration): - # Avoid division by 0 - if timeout_value: - val = duration / timeout_value - timeout_usage.observe(val) + class observe_timeout_usage: + def calculate_timeout_usage(duration): + # Avoid division by 0 + if timeout_value: + val = duration / timeout_value + timeout_usage.observe(val) - timer = context_managers.Timer(observe_timeout_usage) + timer = context_managers.Timer( + observe_timeout_usage, 'calculate_timeout_usage') context = super().get_context(*args, **kwargs) context.enter_context(timer) return context diff --git a/armada/tests/unit/handlers/test_override.py b/armada/tests/unit/handlers/test_override.py index c790a110..f67fee06 100644 --- a/armada/tests/unit/handlers/test_override.py +++ b/armada/tests/unit/handlers/test_override.py @@ -273,7 +273,7 @@ class OverrideTestCase(testtools.TestCase): ovr = Override(documents) ovr.update_documents(merging_values) ovr_doc = ovr.find_manifest_document(doc_path) - expect_doc = list(yaml.load_all(e.read()))[0] + expect_doc = list(yaml.safe_load_all(e.read()))[0] self.assertEqual(ovr_doc, expect_doc) @@ -290,7 +290,7 @@ class OverrideTestCase(testtools.TestCase): ovr = Override(documents, override) ovr.update_manifests() ovr_doc = ovr.find_manifest_document(doc_path) - target_docs = list(yaml.load_all(e.read())) + target_docs = list(yaml.safe_load_all(e.read())) expected_doc = [ x for x in target_docs if x.get('schema') == 'armada/Manifest/v1' diff --git a/doc/requirements.txt b/doc/requirements.txt index e82ef36d..2a1ed4ad 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,9 +3,7 @@ # process, which may cause wedges in the gate later. docutils<=0.17 Jinja2<=3.0.3 -MarkupSafe<2.1.0, >=0.9.2 plantuml -pylibyaml==0.1.0 reno sphinx-rtd-theme==0.5.0 Sphinx==3.3.1 diff --git a/doc/source/_static/armada.conf.sample b/doc/source/_static/armada.conf.sample index 4762ede8..1b337d7d 100644 --- a/doc/source/_static/armada.conf.sample +++ b/doc/source/_static/armada.conf.sample @@ -152,7 +152,7 @@ # Format string to use for log messages with context. Used by # oslo_log.formatters.ContextFormatter (string value) -#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s +#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s # Format string to use for log messages when context is undefined. Used by # oslo_log.formatters.ContextFormatter (string value) @@ -169,7 +169,7 @@ # Defines the format string for %(user_identity)s that is used in # logging_context_format_string. Used by oslo_log.formatters.ContextFormatter # (string value) -#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s +#logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s # List of package logging levels in logger=LEVEL pairs. This option is ignored # if log_config_append is set. (list value) @@ -261,7 +261,7 @@ # Domain name containing project (string value) #project_domain_name = -# ID of the trust to use as a trustee use (string value) +# Trust ID (string value) #trust_id = # Optional domain ID to use with v3 and v2 parameters. It will be used for both diff --git a/images/armada/Dockerfile.ubuntu_focal b/images/armada/Dockerfile.ubuntu_focal index b340f403..8826a56b 100644 --- a/images/armada/Dockerfile.ubuntu_focal +++ b/images/armada/Dockerfile.ubuntu_focal @@ -94,6 +94,8 @@ ENV PBR_VERSION 0.8.0 RUN \ chown -R armada:users . && \ - python3 setup.py install + pip3 install -e . --verbose --use-pep517 \ + && echo "/armada" \ + > /usr/local/lib/python3.8/dist-packages/deckhand.pth USER armada diff --git a/requirements-direct.txt b/requirements-direct.txt index cb7f1563..cbef885f 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -1,36 +1,27 @@ click -configparser -deepdiff<=5.8.1 -docutils<=0.17 +deepdiff falcon -gitpython -importlib-metadata~=2.0 -Jinja2<=3.0.3 -jsonschema<=3.2.0 -keystoneauth1<=5.1.1 -kubernetes==26.1.0 -MarkupSafe<2.1.0, >=0.9.2 -nose -oslo.config<=8.7.1 -oslo.i18n -oslo.log<=4.6.0 -oslo.policy<=3.10.1 -oslo.middleware<=4.4.0 -keystonemiddleware<=10.2.0 -oslo.utils<=4.12.3 -Paste>=2.0.3 -PasteDeploy>=1.5.2 -prometheus-client==0.8.0 -pylibyaml==0.1.0 -python-dateutil==2.8.1 -PyYAML<=5.4.1 -reno -requests==2.23.0 +GitPython +jsonschema +keystoneauth1==4.3.1 +kubernetes +mock +oslo.config==8.6.0 +oslo.context==4.0.0 +oslo.log==4.8.0 +oslo.policy==3.10.1 +oslo.serialization==4.1.0 +oslo.utils==4.12.3 +oslo.db==11.2.0 +packaging +pbr +prometheus-client +pylibyaml +python_dateutil +PyYAML +Requests +responses retry -setuptools<=45.2.0 -six -Sphinx==3.3.1 -sphinx_rtd_theme==0.5.0 -urllib3 >= 1.21.1, <= 1.25.11 -uWSGI==2.0.21 -wheel \ No newline at end of file +testtools +urllib3 +uWSGI \ No newline at end of file diff --git a/requirements-frozen-bionic.txt b/requirements-frozen-bionic.txt deleted file mode 100644 index bfb938e4..00000000 --- a/requirements-frozen-bionic.txt +++ /dev/null @@ -1,92 +0,0 @@ -alabaster==0.7.13 -attrs==22.2.0 -Babel==2.11.0 -bcrypt==4.0.1 -cachetools==4.2.4 -certifi==2023.5.7 -chardet==3.0.4 -click==8.0.4 -configparser==5.2.0 -debtcollector==2.5.0 -decorator==5.1.1 -deepdiff==5.7.0 -docutils==0.17 -dogpile.cache==1.2.0 -dulwich==0.20.50 -falcon==3.1.1 -gitdb==4.0.9 -GitPython==3.1.18 -google-auth==2.18.1 -idna==2.10 -imagesize==1.4.1 -importlib-metadata==2.1.3 -importlib-resources==5.4.0 -iso8601==1.1.0 -Jinja2==3.0.3 -jsonschema==3.2.0 -keystoneauth1==5.1.1 -keystonemiddleware==9.5.0 -kubernetes==26.1.0 -MarkupSafe==2.0.1 -msgpack==1.0.5 -netaddr==0.8.0 -netifaces==0.11.0 -nose==1.3.7 -oauthlib==3.2.2 -ordered-set==4.0.2 -os-service-types==1.7.0 -oslo.cache==2.11.0 -oslo.config==8.7.1 -oslo.context==4.1.0 -oslo.i18n==5.1.0 -oslo.log==4.6.0 -oslo.middleware==4.4.0 -oslo.policy==3.10.1 -oslo.serialization==4.3.0 -oslo.utils==4.12.3 -packaging==21.3 -Paste==3.5.3 -PasteDeploy==2.1.1 -pbr==5.11.1 -pip==21.3.1 -prometheus-client==0.8.0 -py==1.11.0 -pyasn1==0.5.0 -pyasn1-modules==0.3.0 -pycadf==3.1.1 -Pygments==2.14.0 -pylibyaml==0.1.0 -pyparsing==3.0.9 -pyrsistent==0.18.0 -python-dateutil==2.8.1 -python-keystoneclient==4.5.0 -pytz==2023.3 -PyYAML==5.4.1 -reno==4.0.0 -requests==2.23.0 -requests-oauthlib==1.3.1 -retry==0.9.2 -rfc3986==1.5.0 -rsa==4.9 -setuptools==45.2.0 -six==1.16.0 -smmap==5.0.0 -snowballstemmer==2.2.0 -Sphinx==3.3.1 -sphinx-rtd-theme==0.5.0 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.0 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -statsd==4.0.1 -stevedore==3.5.2 -typing_extensions==4.1.1 -urllib3==1.25.11 -uWSGI==2.0.21 -WebOb==1.8.7 -websocket-client==1.3.1 -wheel==0.37.1 -wrapt==1.15.0 -zipp==3.6.0 diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 1664bb60..9e8b1f8c 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,90 +1,78 @@ -alabaster==0.7.13 +alembic==1.11.1 attrs==23.1.0 -Babel==2.12.1 -bcrypt==4.0.1 -cachetools==5.3.0 -certifi==2022.12.7 -chardet==3.0.4 +cachetools==5.3.1 +certifi==2023.5.7 +charset-normalizer==3.1.0 click==8.1.3 -configparser==5.3.0 debtcollector==2.5.0 decorator==5.1.1 -deepdiff==5.8.1 -docutils==0.17 -dogpile.cache==1.2.0 -dulwich==0.21.3 +deepdiff==6.3.1 falcon==3.1.1 +fixtures==4.1.0 gitdb==4.0.10 GitPython==3.1.31 -google-auth==2.17.3 -idna==2.10 -imagesize==1.4.1 -importlib-metadata==2.1.3 -iso8601==1.1.0 -Jinja2==3.0.3 -jsonschema==3.2.0 -keystoneauth1==5.1.1 -keystonemiddleware==10.2.0 -kubernetes==26.1.0 -MarkupSafe==2.0.1 +google-auth==2.20.0 +greenlet==2.0.2 +idna==3.4 +importlib-metadata==4.13.0 +importlib-resources==5.12.0 +iso8601==2.0.0 +jsonschema==4.17.3 +keystoneauth1==4.3.1 +kubernetes==23.6.0 +Mako==1.2.4 +MarkupSafe==2.1.3 +mock==5.1.0 msgpack==1.0.5 netaddr==0.8.0 netifaces==0.11.0 -nose==1.3.7 oauthlib==3.2.2 ordered-set==4.1.0 os-service-types==1.7.0 -oslo.cache==3.3.1 -oslo.config==8.7.1 -oslo.context==5.1.1 +oslo.config==8.6.0 +oslo.context==4.0.0 +oslo.db==11.2.0 oslo.i18n==6.0.0 -oslo.log==4.6.0 -oslo.middleware==4.4.0 +oslo.log==4.8.0 oslo.policy==3.10.1 -oslo.serialization==5.1.1 +oslo.serialization==4.1.0 oslo.utils==4.12.3 -packaging==23.1 -Paste==3.5.3 -PasteDeploy==3.0.1 +packaging==21.3 pbr==5.11.1 -pip==23.0.1 -prometheus-client==0.8.0 +pip==23.1.2 +pkgutil_resolve_name==1.3.10 +prometheus-client==0.17.0 py==1.11.0 -pyasn1==0.5.0 -pyasn1-modules==0.3.0 -pycadf==3.1.1 -Pygments==2.15.1 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 pylibyaml==0.1.0 pyparsing==3.0.9 pyrsistent==0.19.3 -python-dateutil==2.8.1 -python-keystoneclient==5.1.0 +python-dateutil==2.8.2 pytz==2023.3 -PyYAML==5.4.1 -reno==4.0.0 -requests==2.23.0 +PyYAML==6.0 +requests==2.31.0 requests-oauthlib==1.3.1 +responses==0.23.1 retry==0.9.2 -rfc3986==2.0.0 +rfc3986==1.5.0 rsa==4.9 -setuptools==45.2.0 +setuptools==67.7.2 six==1.16.0 smmap==5.0.0 -snowballstemmer==2.2.0 -Sphinx==3.3.1 -sphinx-rtd-theme==0.5.0 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.5 -statsd==4.0.1 -stevedore==5.0.0 -urllib3==1.25.11 +SQLAlchemy==1.4.48 +sqlalchemy-migrate==0.13.0 +sqlparse==0.4.4 +stevedore==5.1.0 +Tempita==0.5.2 +testresources==2.0.1 +testscenarios==0.5.0 +testtools==2.6.0 +types-PyYAML==6.0.12.10 +typing_extensions==4.6.3 +urllib3==1.26.16 uWSGI==2.0.21 -WebOb==1.8.7 -websocket-client==1.5.1 +websocket-client==1.5.3 wheel==0.40.0 wrapt==1.15.0 zipp==3.15.0 diff --git a/test-requirements.txt b/test-requirements.txt index b6e17e86..84af7c85 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,20 +1,17 @@ bandit==1.6.0 -cmd2>=1.5.0 +# cmd2>=1.5.0 coverage==5.3 flake8-import-order>=0.18.1 flake8==3.8.4 grpcio-tools>=1.16.0 -mock==4.0.2 os-testr>=1.0.0 # Apache-2.0 pyflakes==2.2.0 -responses==0.12.1 sphinx-rtd-theme==0.5.0 stestr==3.2.0 testrepository==0.0.20 testresources==2.0.1 testscenarios==0.5.0 -testtools==2.5.0 -testtools==2.5.0 -tox>=3.28.0, <4.0.0 -typing-extensions>=3.7.2 +# testtools==2.5.0 +# tox>=3.28.0, <4.0.0 +# typing-extensions>=3.7.2 yapf==0.27.0 \ No newline at end of file diff --git a/tools/gate/playbooks/airskiff-deploy.yaml b/tools/gate/playbooks/airskiff-deploy.yaml index c4eb2e6f..e4d9440a 100644 --- a/tools/gate/playbooks/airskiff-deploy.yaml +++ b/tools/gate/playbooks/airskiff-deploy.yaml @@ -48,11 +48,7 @@ set -ex export DISTRO={{ ARMADA_IMAGE_DISTRO }} make images - if test "${DISTRO}" = 'ubuntu_bionic' - then - # this trick is needed to use bionic image instead of focal in airskiff deployment test - docker tag quay.io/airshipit/armada:latest-ubuntu_bionic quay.io/airshipit/armada:latest-ubuntu_focal - fi + docker system prune --force args: chdir: "{{ zuul.project.src_dir }}" become: yes @@ -65,6 +61,13 @@ chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" become: yes + - name: Setup openstack client + shell: | + ./tools/deployment/airskiff/developer/020-setup-client.sh + args: + chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}" + become: yes + - name: Deploy Airship components using Armada shell: | mkdir -p ~/.kube diff --git a/tox.ini b/tox.ini index 3fc5b584..8ce68e7c 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,6 @@ commands = commands = {posargs} - [testenv:freeze] basepython=python3 recreate = True @@ -39,33 +38,12 @@ allowlist_externals= sh deps= -r{toxinidir}/requirements-direct.txt + -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.2/constraints-3.8.txt commands= rm -f requirements-frozen.txt sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen.txt" -[testenv:freeze-bionic] -basepython=python3 -recreate = True -allowlist_externals= - rm - sh -deps= - -r{toxinidir}/requirements-direct.txt -commands= - rm -f requirements-frozen-bionic.txt - sh -c "pip freeze --all | grep -vE 'armada|pyinotify|pkg-resources==0.0.0' > requirements-frozen-bionic.txt" - - -[testenv:py36] -deps= - -r{toxinidir}/requirements-frozen-bionic.txt - -r{toxinidir}/test-requirements.txt -commands = - {[testenv]commands} - stestr run {posargs} - stestr slowest - [testenv:py38] commands = {[testenv]commands}