From cba78d1d03e4910f6ab1691bae633c5bddce893d Mon Sep 17 00:00:00 2001 From: gardlt Date: Mon, 18 Sep 2017 00:24:10 +0000 Subject: [PATCH] style(armada): converting py2 to py3 - Format code to python 3.5 - using absoulute paths Change-Id: I7414b5de915429c2c7f85b99f2ab91f395c62121 --- Dockerfile | 14 +-- armada/api/server.py | 19 ++- armada/exceptions/api_exceptions.py | 2 +- armada/exceptions/armada_exceptions.py | 2 +- armada/exceptions/chartbuilder_exceptions.py | 2 +- armada/exceptions/lint_exceptions.py | 2 +- armada/exceptions/source_exceptions.py | 2 +- armada/exceptions/tiller_exceptions.py | 2 +- armada/handlers/armada.py | 113 +++++++++--------- armada/handlers/chartbuilder.py | 25 ++-- armada/handlers/manifest.py | 2 +- armada/handlers/tiller.py | 14 +-- armada/tests/unit/api/test_api.py | 4 +- .../tests/unit/handlers/test_chartbuilder.py | 3 +- requirements.txt | 4 +- test-requirements.txt | 4 +- tox.ini | 3 +- 17 files changed, 107 insertions(+), 110 deletions(-) diff --git a/Dockerfile b/Dockerfile index b0c9e65d..6baeaa7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,21 +10,21 @@ RUN apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ netbase \ - python-all \ - python-pip \ - python-setuptools && \ + python3-pip && \ apt-get install -y \ build-essential \ curl \ git \ - python-all-dev && \ + python3-minimal \ + python3-setuptools \ + python3-dev && \ useradd -u 1000 -g users -d /armada armada && \ chown -R armada:users /armada && \ \ cd /armada && \ - pip install --upgrade pip && \ - pip install -r requirements.txt && \ - pip install . && \ + pip3 install --upgrade pip && \ + pip3 install -r requirements.txt && \ + python3 setup.py install && \ \ apt-get purge --auto-remove -y \ build-essential \ diff --git a/armada/api/server.py b/armada/api/server.py index 3e13e7cf..cdc76d9f 100644 --- a/armada/api/server.py +++ b/armada/api/server.py @@ -12,23 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import falcon import os +import falcon from oslo_config import cfg from oslo_log import log as logging -from armada.common import policy from armada import conf - from armada.api import ArmadaRequest -from armada_controller import Apply -from middleware import AuthMiddleware -from middleware import ContextMiddleware -from middleware import LoggingMiddleware -from tiller_controller import Release -from tiller_controller import Status -from validation_controller import Validate +from armada.api.armada_controller import Apply +from armada.api.middleware import AuthMiddleware +from armada.api.middleware import ContextMiddleware +from armada.api.middleware import LoggingMiddleware +from armada.api.tiller_controller import Release +from armada.api.tiller_controller import Status +from armada.api.validation_controller import Validate +from armada.common import policy LOG = logging.getLogger(__name__) conf.set_app_default_configs() diff --git a/armada/exceptions/api_exceptions.py b/armada/exceptions/api_exceptions.py index a5c8838a..ffec860e 100644 --- a/armada/exceptions/api_exceptions.py +++ b/armada/exceptions/api_exceptions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base_exception as base +from armada.exceptions import base_exception as base class ApiException(base.ArmadaBaseException): diff --git a/armada/exceptions/armada_exceptions.py b/armada/exceptions/armada_exceptions.py index 3aec0c96..82fe9e72 100644 --- a/armada/exceptions/armada_exceptions.py +++ b/armada/exceptions/armada_exceptions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base_exception +from armada.exceptions import base_exception class ArmadaException(base_exception.ArmadaBaseException): diff --git a/armada/exceptions/chartbuilder_exceptions.py b/armada/exceptions/chartbuilder_exceptions.py index 2229df0b..5b2d1836 100644 --- a/armada/exceptions/chartbuilder_exceptions.py +++ b/armada/exceptions/chartbuilder_exceptions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base_exception +from armada.exceptions import base_exception class ChartBuilderException(base_exception.ArmadaBaseException): diff --git a/armada/exceptions/lint_exceptions.py b/armada/exceptions/lint_exceptions.py index 8ca2276b..aa0ead76 100644 --- a/armada/exceptions/lint_exceptions.py +++ b/armada/exceptions/lint_exceptions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base_exception +from armada.exceptions import base_exception class LintException(base_exception.ArmadaBaseException): diff --git a/armada/exceptions/source_exceptions.py b/armada/exceptions/source_exceptions.py index dc9318d5..97dc22d6 100644 --- a/armada/exceptions/source_exceptions.py +++ b/armada/exceptions/source_exceptions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base_exception +from armada.exceptions import base_exception class SourceException(base_exception.ArmadaBaseException): diff --git a/armada/exceptions/tiller_exceptions.py b/armada/exceptions/tiller_exceptions.py index 76aea982..6c187f01 100644 --- a/armada/exceptions/tiller_exceptions.py +++ b/armada/exceptions/tiller_exceptions.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from base_exception import ArmadaBaseException as ex +from armada.exceptions.base_exception import ArmadaBaseException as ex class TillerException(ex): diff --git a/armada/handlers/armada.py b/armada/handlers/armada.py index b5d882bf..2e2c891d 100644 --- a/armada/handlers/armada.py +++ b/armada/handlers/armada.py @@ -18,20 +18,17 @@ import yaml from oslo_log import log as logging from supermutes.dot import dotify -from chartbuilder import ChartBuilder -from tiller import Tiller -from manifest import Manifest - -from ..exceptions import armada_exceptions -from ..exceptions import source_exceptions -from ..exceptions import lint_exceptions -from ..exceptions import tiller_exceptions - -from ..utils.release import release_prefix -from ..utils import source -from ..utils import lint -from ..const import KEYWORD_ARMADA, KEYWORD_GROUPS, KEYWORD_CHARTS,\ - KEYWORD_PREFIX, STATUS_FAILED +from armada.handlers.chartbuilder import ChartBuilder +from armada.handlers.tiller import Tiller +from armada.handlers.manifest import Manifest +from armada.exceptions import armada_exceptions +from armada.exceptions import source_exceptions +from armada.exceptions import lint_exceptions +from armada.exceptions import tiller_exceptions +from armada.utils.release import release_prefix +from armada.utils import source +from armada.utils import lint +from armada import const LOG = logging.getLogger(__name__) @@ -105,11 +102,13 @@ class Armada(object): self.config = self.get_armada_manifest() # Purge known releases that have failed and are in the current yaml - prefix = self.config.get(KEYWORD_ARMADA).get(KEYWORD_PREFIX) - failed_releases = self.get_releases_by_status(STATUS_FAILED) + prefix = self.config.get(const.KEYWORD_ARMADA).get( + const.KEYWORD_PREFIX) + failed_releases = self.get_releases_by_status(const.STATUS_FAILED) for release in failed_releases: - for group in self.config.get(KEYWORD_ARMADA).get(KEYWORD_GROUPS): - for ch in group.get(KEYWORD_CHARTS): + for group in self.config.get(const.KEYWORD_ARMADA).get( + const.KEYWORD_GROUPS): + for ch in group.get(const.KEYWORD_CHARTS): ch_release_name = release_prefix(prefix, ch.get('chart') .get('chart_name')) @@ -123,8 +122,9 @@ class Armada(object): # We only support a git source type right now, which can also # handle git:// local paths as well repos = {} - for group in self.config.get(KEYWORD_ARMADA).get(KEYWORD_GROUPS): - for ch in group.get(KEYWORD_CHARTS): + for group in self.config.get(const.KEYWORD_ARMADA).get( + const.KEYWORD_GROUPS): + for ch in group.get(const.KEYWORD_CHARTS): self.tag_cloned_repo(ch, repos) for dep in ch.get('chart').get('dependencies'): @@ -142,11 +142,11 @@ class Armada(object): tarball_dir = source.get_tarball(location) ch.get('chart')['source_dir'] = (tarball_dir, subpath) elif ct_type == 'git': - reference = ch.get('chart').get('source').get('reference', - 'master') + reference = ch.get('chart').get('source').get( + 'reference', 'master') repo_branch = (location, reference) - if repo_branch not in repos.keys(): + if repo_branch not in repos: try: LOG.info('Cloning repo: %s branch: %s', *repo_branch) repo_dir = source.git_clone(*repo_branch) @@ -181,11 +181,7 @@ class Armada(object): Syncronize Helm with the Armada Config(s) ''' - msg = { - 'installed': [], - 'upgraded': [], - 'diff': [] - } + msg = {'installed': [], 'upgraded': [], 'diff': []} # TODO: (gardlt) we need to break up this func into # a more cleaner format @@ -194,7 +190,8 @@ class Armada(object): # extract known charts on tiller right now known_releases = self.tiller.list_charts() - prefix = self.config.get(KEYWORD_ARMADA).get(KEYWORD_PREFIX) + prefix = self.config.get(const.KEYWORD_ARMADA).get( + const.KEYWORD_PREFIX) if known_releases is None: raise armada_exceptions.KnownReleasesException() @@ -203,11 +200,11 @@ class Armada(object): LOG.debug("Release %s, Version %s found on tiller", release[0], release[1]) - for entry in self.config[KEYWORD_ARMADA][KEYWORD_GROUPS]: + for entry in self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]: chart_wait = self.wait desc = entry.get('description', 'A Chart Group') - chart_group = entry.get(KEYWORD_CHARTS, []) + chart_group = entry.get(const.KEYWORD_CHARTS, []) test_charts = entry.get('test_charts', False) if entry.get('sequenced', False) or test_charts: @@ -273,8 +270,8 @@ class Armada(object): # once we support those upgrade_diff = self.show_diff( - chart, apply_chart, apply_values, chartbuilder.dump(), - values, msg) + chart, apply_chart, apply_values, + chartbuilder.dump(), values, msg) if not upgrade_diff: LOG.info("There are no updates found in this chart") @@ -282,30 +279,31 @@ class Armada(object): # do actual update LOG.info('wait: %s', chart_wait) - self.tiller.update_release(protoc_chart, - prefix_chart, - chart.namespace, - pre_actions=pre_actions, - post_actions=post_actions, - dry_run=self.dry_run, - disable_hooks=chart. - upgrade.no_hooks, - values=yaml.safe_dump(values), - wait=chart_wait, - timeout=chart_timeout) + self.tiller.update_release( + protoc_chart, + prefix_chart, + chart.namespace, + pre_actions=pre_actions, + post_actions=post_actions, + dry_run=self.dry_run, + disable_hooks=chart.upgrade.no_hooks, + values=yaml.safe_dump(values), + wait=chart_wait, + timeout=chart_timeout) msg['upgraded'].append(prefix_chart) # process install else: LOG.info("Installing release %s", chart.release) - self.tiller.install_release(protoc_chart, - prefix_chart, - chart.namespace, - dry_run=self.dry_run, - values=yaml.safe_dump(values), - wait=chart_wait, - timeout=chart_timeout) + self.tiller.install_release( + protoc_chart, + prefix_chart, + chart.namespace, + dry_run=self.dry_run, + values=yaml.safe_dump(values), + wait=chart_wait, + timeout=chart_timeout) msg['installed'].append(prefix_chart) @@ -328,7 +326,8 @@ class Armada(object): if self.enable_chart_cleanup: self.tiller.chart_cleanup( - prefix, self.config[KEYWORD_ARMADA][KEYWORD_GROUPS]) + prefix, + self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]) return msg @@ -337,8 +336,9 @@ class Armada(object): Operations to run after deployment process has terminated ''' # Delete temp dirs used for deployment - for group in self.config.get(KEYWORD_ARMADA).get(KEYWORD_GROUPS): - for ch in group.get(KEYWORD_CHARTS): + for group in self.config.get(const.KEYWORD_ARMADA).get( + const.KEYWORD_GROUPS): + for ch in group.get(const.KEYWORD_CHARTS): if ch.get('chart').get('source').get('type') == 'git': source.source_cleanup(ch.get('chart').get('source_dir')[0]) @@ -351,10 +351,9 @@ class Armada(object): unified diff output and avoid the use of print ''' + source = str(installed_chart.SerializeToString()).split('\n') chart_diff = list( - difflib.unified_diff( - installed_chart.SerializeToString().split('\n'), - target_chart.split('\n'))) + difflib.unified_diff(source, str(target_chart).split('\n'))) if len(chart_diff) > 0: LOG.info("Chart Unified Diff (%s)", chart.release) diff --git a/armada/handlers/chartbuilder.py b/armada/handlers/chartbuilder.py index 43fde09d..c82a1b0d 100644 --- a/armada/handlers/chartbuilder.py +++ b/armada/handlers/chartbuilder.py @@ -15,13 +15,13 @@ import os import yaml -from hapi.chart.template_pb2 import Template from hapi.chart.chart_pb2 import Chart -from hapi.chart.metadata_pb2 import Metadata from hapi.chart.config_pb2 import Config +from hapi.chart.metadata_pb2 import Metadata +from hapi.chart.template_pb2 import Template from supermutes.dot import dotify -from ..exceptions import chartbuilder_exceptions +from armada.exceptions import chartbuilder_exceptions from oslo_config import cfg from oslo_log import log as logging @@ -108,10 +108,9 @@ class ChartBuilder(object): # extract Chart.yaml to construct metadata try: - chart_yaml = dotify( - yaml.safe_load( - open(os.path.join(self.source_directory, 'Chart.yaml')) - .read())) + with open(os.path.join(self.source_directory, 'Chart.yaml')) as f: + chart_yaml = dotify(yaml.safe_load(f.read().encode('utf-8'))) + except Exception: raise chartbuilder_exceptions.MetadataLoadException() @@ -136,8 +135,8 @@ class ChartBuilder(object): # create config object representing unmarshaled values.yaml if os.path.exists(os.path.join(self.source_directory, 'values.yaml')): - raw_values = open( - os.path.join(self.source_directory, 'values.yaml')).read() + with open(os.path.join(self.source_directory, 'values.yaml')) as f: + raw_values = f.read() else: LOG.warn("No values.yaml in %s, using empty values", self.source_directory) @@ -168,10 +167,10 @@ class ChartBuilder(object): LOG.debug('Ignoring file %s', tname) continue - templates.append( - Template( - name=tname, - data=open(os.path.join(root, tpl_file), 'r').read())) + with open(os.path.join(root, tpl_file)) as f: + templates.append( + Template(name=tname, data=f.read().encode())) + return templates def get_helm_chart(self): diff --git a/armada/handlers/manifest.py b/armada/handlers/manifest.py index 2a88b3ba..fed998af 100644 --- a/armada/handlers/manifest.py +++ b/armada/handlers/manifest.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ..const import DOCUMENT_CHART, DOCUMENT_GROUP, DOCUMENT_MANIFEST +from armada.const import DOCUMENT_CHART, DOCUMENT_GROUP, DOCUMENT_MANIFEST class Manifest(object): diff --git a/armada/handlers/tiller.py b/armada/handlers/tiller.py index 48012baf..59c17a92 100644 --- a/armada/handlers/tiller.py +++ b/armada/handlers/tiller.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import yaml import grpc +import yaml from hapi.chart.config_pb2 import Config from hapi.services.tiller_pb2 import GetReleaseContentRequest @@ -28,10 +28,10 @@ from hapi.services.tiller_pb2 import UpdateReleaseRequest from oslo_config import cfg from oslo_log import log as logging -from ..const import STATUS_DEPLOYED, STATUS_FAILED -from ..exceptions import tiller_exceptions as ex -from ..utils.release import release_prefix -from k8s import K8s +from armada.const import STATUS_DEPLOYED, STATUS_FAILED +from armada.exceptions import tiller_exceptions as ex +from armada.handlers.k8s import K8s +from armada.utils.release import release_prefix TILLER_PORT = 44134 @@ -238,8 +238,8 @@ class Tiller(object): label_selector = 'release_name={}'.format(release_name) for label in resource_labels: - label_selector += ', {}={}'.format(label.keys()[0], - label.values()[0]) + label_selector += ', {}={}'.format(list(label.keys())[0], + list(label.values())[0]) if 'job' in resource_type: LOG.info("Deleting %s in namespace: %s", resource_name, namespace) diff --git a/armada/tests/unit/api/test_api.py b/armada/tests/unit/api/test_api.py index 06c12668..56eb8f24 100644 --- a/armada/tests/unit/api/test_api.py +++ b/armada/tests/unit/api/test_api.py @@ -18,11 +18,13 @@ import unittest import falcon from falcon import testing +from oslo_config import cfg -from armada import conf as cfg +from armada import conf from armada.api import server CONF = cfg.CONF +conf.set_app_default_configs() class APITestCase(testing.TestCase): diff --git a/armada/tests/unit/handlers/test_chartbuilder.py b/armada/tests/unit/handlers/test_chartbuilder.py index 71138f83..12549ab6 100644 --- a/armada/tests/unit/handlers/test_chartbuilder.py +++ b/armada/tests/unit/handlers/test_chartbuilder.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import unittest import mock @@ -83,4 +84,4 @@ class ChartBuilderTestCase(unittest.TestCase): resp = chartbuilder.get_metadata() self.assertIsNotNone(resp) - self.assertIsInstance(resp, basestring) + self.assertIsInstance(resp, str) diff --git a/requirements.txt b/requirements.txt index d4fe8586..6c01feab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,16 +7,14 @@ kubernetes>=1.0.0 protobuf==3.2.0 PyYAML==3.12 requests==2.17.3 -sphinx_rtd_theme supermutes==0.2.5 urllib3==1.21.1 -uwsgi>=2.0.15 Paste>=2.0.3 PasteDeploy>=1.5.2 # API falcon==1.1.0 -gunicorn==19.7.1 +uwsgi>=2.0.15 # CLI cliff==2.7.0 diff --git a/test-requirements.txt b/test-requirements.txt index af6db1d7..42c4c8df 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,10 +3,10 @@ tox # Docs sphinx>=1.6.2 -sphinx_rtd_theme==0.2.4 +sphinx_rtd_theme>=0.2.4 # Testing -flake8==3.3.0 +flake8>=3.3.0 nose==1.3.7 testtools==2.3.0 codecov diff --git a/tox.ini b/tox.ini index 5f50f479..e6c6a193 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,8 @@ [tox] skipsdist = True -envlist = py27, pep8, coverage, bandit +envlist = py35, pep8, coverage, bandit [testenv] -basepython=python2.7 deps= -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt