[feat] adding standard armada manifest

- adding tmeplate armada manifest
- create Armada Manifest
- updated validation for new documents
- updated testing
- updated docs
This commit is contained in:
gardlt 2017-07-20 10:14:01 -05:00 committed by Alexis Rivera DeLa Torre
parent f8ff2c32df
commit afb7fe83ab
19 changed files with 1043 additions and 518 deletions

View File

@ -13,10 +13,24 @@
# limitations under the License.
#
from armada.handlers.armada import Armada as Handler
from falcon import HTTP_200
from armada.conf import default
import json
from falcon import HTTP_200
from oslo_config import cfg
from oslo_log import log as logging
# Required Oslo configuration setup
default.register_opts()
from armada.handlers.armada import Armada as Handler
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
DOMAIN = "armada"
logging.setup(CONF, DOMAIN)
class Apply(object):
'''

View File

@ -15,7 +15,8 @@
from cliff import command as cmd
import yaml
from armada.utils.lint import valid_manifest
from armada.utils.lint import validate_armada_documents, validate_armada_object
from armada.handlers.manifest import Manifest
from oslo_config import cfg
from oslo_log import log as logging
@ -28,9 +29,16 @@ DOMAIN = "armada"
logging.setup(CONF, DOMAIN)
def validateYaml(args):
config = yaml.load(open(args.file).read())
if valid_manifest(config):
LOG.info('File successfully validated')
documents = yaml.safe_load_all(open(args.file).read())
manifest_obj = Manifest(documents).get_manifest()
obj_check = validate_armada_object(manifest_obj)
doc_check = validate_armada_documents(documents)
try:
if doc_check and obj_check:
LOG.info('Successfully validated: %s', args.file)
except Exception:
raise Exception('Failed to validate: %s', args.file)
class ValidateYamlCommand(cmd.Command):
def get_parser(self, prog_name):

25
armada/const.py Normal file
View File

@ -0,0 +1,25 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Documents
DOCUMENT_CHART = 'armada/Chart/v1'
DOCUMENT_GROUP = 'armada/ChartGroup/v1'
DOCUMENT_MANIFEST = 'armada/Manifest/v1'
# Armada Data
KEYWORD_ARMADA = 'armada'
KEYWORD_PREFIX = 'release_prefix'
KEYWORD_GROUPS = 'chart_groups'
KEYWORD_CHARTS = 'chart_group'
KEYWORD_RELEASE = 'release'

View File

@ -21,9 +21,12 @@ from supermutes.dot import dotify
from chartbuilder import ChartBuilder
from tiller import Tiller
from manifest import Manifest
from ..utils.release import release_prefix
from ..utils import git
from ..utils import lint
from ..const import KEYWORD_ARMADA, KEYWORD_GROUPS, KEYWORD_CHARTS,\
KEYWORD_PREFIX
LOG = logging.getLogger(__name__)
@ -33,13 +36,15 @@ DOMAIN = "armada"
logging.setup(CONF, DOMAIN)
class Armada(object):
'''
This is the main Armada class handling the Armada
workflows
'''
def __init__(self, config,
def __init__(self,
file,
disable_update_pre=False,
disable_update_post=False,
enable_chart_cleanup=False,
@ -53,20 +58,26 @@ class Armada(object):
Initialize the Armada Engine and establish
a connection to Tiller
'''
self.disable_update_pre = disable_update_pre
self.disable_update_post = disable_update_post
self.enable_chart_cleanup = enable_chart_cleanup
self.dry_run = dry_run
self.wait = wait
self.timeout = timeout
self.config = yaml.load(config)
self.tiller = Tiller(tiller_host=tiller_host, tiller_port=tiller_port)
self.documents = yaml.safe_load_all(file)
self.config = self.get_armada_manifest()
self.tiller = Tiller()
self.debug = debug
# Set debug value
CONF.set_default('debug', self.debug)
logging.setup(CONF, DOMAIN)
def get_armada_manifest(self):
return Manifest(self.documents).get_manifest()
def find_release_chart(self, known_releases, name):
'''
Find a release given a list of known_releases and a release name
@ -79,11 +90,14 @@ class Armada(object):
'''
Perform a series of checks and operations to ensure proper deployment
'''
# Ensure tiller is available and yaml is valid
if not self.tiller.tiller_status():
raise Exception("Tiller Services is not Available")
if not lint.valid_manifest(self.config):
raise Exception("Service: Tiller is not Available")
if not lint.validate_armada_documents(self.documents):
raise Exception("Invalid Armada Manifest")
if not lint.validate_armada_object(self.config):
raise Exception("Invalid Armada Object")
# Purge known releases that have failed and are in the current yaml
prefix = self.config.get('armada').get('release_prefix')
@ -91,8 +105,9 @@ class Armada(object):
for release in failed_releases:
for group in self.config.get('armada').get('charts'):
for ch in group.get('chart_group'):
ch_release_name = release_prefix(prefix, ch.get('chart')
.get('name'))
ch_release_name = release_prefix(prefix,
ch.get('chart')
.get('name'))
if release[0] == ch_release_name:
LOG.info('Purging failed release %s '
'before deployment', release[0])
@ -103,30 +118,35 @@ class Armada(object):
# We only support a git source type right now, which can also
# handle git:// local paths as well
repos = {}
for group in self.config.get('armada').get('charts'):
for ch in group.get('chart_group'):
location = ch.get('chart').get('source').get('location')
ct_type = ch.get('chart').get('source').get('type')
reference = ch.get('chart').get('source').get('reference')
subpath = ch.get('chart').get('source').get('subpath')
for group in self.config.get(KEYWORD_ARMADA).get(KEYWORD_GROUPS):
for ch in group.get(KEYWORD_CHARTS):
self.tag_cloned_repo(ch, repos)
if ct_type == 'local':
ch.get('chart')['source_dir'] = (location, subpath)
elif ct_type == 'git':
if location not in repos.keys():
try:
LOG.info('Cloning repo: %s', location)
repo_dir = git.git_clone(location, reference)
except Exception as e:
raise ValueError(e)
repos[location] = repo_dir
ch.get('chart')['source_dir'] = (repo_dir, subpath)
else:
ch.get('chart')['source_dir'] = (repos.get(location),
subpath)
else:
raise Exception("Unknown source type %s for chart %s",
ct_type, ch.get('chart').get('name'))
for dep in ch.get('chart').get('dependencies'):
self.tag_cloned_repo(dep, repos)
def tag_cloned_repo(self, ch, repos):
location = ch.get('chart').get('source').get('location')
ct_type = ch.get('chart').get('source').get('type')
reference = ch.get('chart').get('source').get('reference')
subpath = ch.get('chart').get('source').get('subpath')
if ct_type == 'local':
ch.get('chart')['source_dir'] = (location, subpath)
elif ct_type == 'git':
if location not in repos.keys():
try:
LOG.info('Cloning repo: %s', location)
repo_dir = git.git_clone(location, reference)
except Exception as e:
raise ValueError(e)
repos[location] = repo_dir
ch.get('chart')['source_dir'] = (repo_dir, subpath)
else:
ch.get('chart')['source_dir'] = (repos.get(location), subpath)
else:
raise Exception("Unknown source type %s for chart %s", ct_type,
ch.get('chart').get('name'))
def get_releases_by_status(self, status):
'''
@ -154,16 +174,16 @@ class Armada(object):
# extract known charts on tiller right now
known_releases = self.tiller.list_charts()
prefix = self.config.get('armada').get('release_prefix')
prefix = self.config.get(KEYWORD_ARMADA).get(KEYWORD_PREFIX)
for release in known_releases:
LOG.debug("Release %s, Version %s found on tiller", release[0],
release[1])
for entry in self.config['armada']['charts']:
for entry in self.config[KEYWORD_ARMADA][KEYWORD_GROUPS]:
chart_wait = self.wait
desc = entry.get('description', 'A Chart Group')
chart_group = entry.get('chart_group', [])
chart_group = entry.get(KEYWORD_CHARTS, [])
if entry.get('sequenced', False):
chart_wait = True
@ -175,9 +195,9 @@ class Armada(object):
values = gchart.get('chart').get('values', {})
pre_actions = {}
post_actions = {}
LOG.info('%s', chart.release_name)
LOG.info('%s', chart.release)
if chart.release_name is None:
if chart.release is None:
continue
# retrieve appropriate timeout value if 'wait' is specified
@ -191,14 +211,14 @@ class Armada(object):
protoc_chart = chartbuilder.get_helm_chart()
# determine install or upgrade by examining known releases
LOG.debug("RELEASE: %s", chart.release_name)
LOG.debug("RELEASE: %s", chart.release)
deployed_releases = [x[0] for x in known_releases]
prefix_chart = release_prefix(prefix, chart.release_name)
prefix_chart = release_prefix(prefix, chart.release)
if prefix_chart in deployed_releases:
# indicate to the end user what path we are taking
LOG.info("Upgrading release %s", chart.release_name)
LOG.info("Upgrading release %s", chart.release)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
LOG.info("Checking Pre/Post Actions")
@ -209,12 +229,12 @@ class Armada(object):
upgrade = gchart.get('chart', {}).get('upgrade', False)
if upgrade:
if not self.disable_update_pre and upgrade.get('pre',
False):
if not self.disable_update_pre and upgrade.get(
'pre', False):
pre_actions = getattr(chart.upgrade, 'pre', {})
if not self.disable_update_post and upgrade.get('post',
False):
if not self.disable_update_post and upgrade.get(
'post', False):
post_actions = getattr(chart.upgrade, 'post', {})
# show delta for both the chart templates and the chart
@ -231,29 +251,31 @@ class Armada(object):
continue
# do actual update
self.tiller.update_release(protoc_chart,
self.dry_run,
chart.release_name,
chart.namespace,
prefix, pre_actions,
post_actions,
disable_hooks=chart.
upgrade.no_hooks,
values=yaml.safe_dump(values),
wait=chart_wait,
timeout=chart_timeout)
self.tiller.update_release(
protoc_chart,
self.dry_run,
chart.release,
chart.namespace,
prefix,
pre_actions,
post_actions,
disable_hooks=chart.upgrade.no_hooks,
values=yaml.safe_dump(values),
wait=chart_wait,
timeout=chart_timeout)
# process install
else:
LOG.info("Installing release %s", chart.release_name)
self.tiller.install_release(protoc_chart,
self.dry_run,
chart.release_name,
chart.namespace,
prefix,
values=yaml.safe_dump(values),
wait=chart_wait,
timeout=chart_timeout)
LOG.info("Installing release %s", chart.release)
self.tiller.install_release(
protoc_chart,
self.dry_run,
chart.release,
chart.namespace,
prefix,
values=yaml.safe_dump(values),
wait=chart_wait,
timeout=chart_timeout)
LOG.debug("Cleaning up chart source in %s",
chartbuilder.source_directory)
@ -262,7 +284,8 @@ class Armada(object):
self.post_flight_ops()
if self.enable_chart_cleanup:
self.tiller.chart_cleanup(prefix, self.config['armada']['charts'])
self.tiller.chart_cleanup(
prefix, self.config[KEYWORD_ARMADA][KEYWORD_GROUPS])
def post_flight_ops(self):
'''
@ -274,8 +297,8 @@ class Armada(object):
if ch.get('chart').get('source').get('type') == 'git':
git.source_cleanup(ch.get('chart').get('source_dir')[0])
def show_diff(self, chart, installed_chart,
installed_values, target_chart, target_values):
def show_diff(self, chart, installed_chart, installed_values, target_chart,
target_values):
'''
Produce a unified diff of the installed chart vs our intention
@ -283,20 +306,19 @@ class Armada(object):
unified diff output and avoid the use of print
'''
chart_diff = list(difflib.unified_diff(installed_chart
.SerializeToString()
.split('\n'),
target_chart.split('\n')))
chart_diff = list(
difflib.unified_diff(installed_chart.SerializeToString()
.split('\n'), target_chart.split('\n')))
if len(chart_diff) > 0:
LOG.info("Chart Unified Diff (%s)", chart.release_name)
LOG.info("Chart Unified Diff (%s)", chart.release)
for line in chart_diff:
LOG.debug(line)
values_diff = list(difflib.unified_diff(installed_values.split('\n'),
yaml
.safe_dump(target_values)
.split('\n')))
values_diff = list(
difflib.unified_diff(
installed_values.split('\n'),
yaml.safe_dump(target_values).split('\n')))
if len(values_diff) > 0:
LOG.info("Values Unified Diff (%s)", chart.release_name)
LOG.info("Values Unified Diff (%s)", chart.release)
for line in values_diff:
LOG.debug(line)

View File

@ -31,6 +31,7 @@ DOMAIN = "armada"
logging.setup(CONF, DOMAIN)
class ChartBuilder(object):
'''
This class handles taking chart intentions as a paramter and
@ -69,8 +70,7 @@ class ChartBuilder(object):
'''
Return the joined path of the source directory and subpath
'''
return os.path.join(self.chart.source_dir[0],
self.chart.source_dir[1])
return os.path.join(self.chart.source_dir[0], self.chart.source_dir[1])
def get_ignored_files(self):
'''
@ -90,8 +90,8 @@ class ChartBuilder(object):
false otherwise
'''
for ignored_file in self.ignored_files:
if (ignored_file.startswith('*') and
filename.endswith(ignored_file.strip('*'))):
if (ignored_file.startswith('*')
and filename.endswith(ignored_file.strip('*'))):
return True
elif ignored_file == filename:
return True
@ -102,15 +102,16 @@ class ChartBuilder(object):
Process metadata
'''
# extract Chart.yaml to construct metadata
chart_yaml = dotify(yaml.load(open(os.path.join(self.source_directory,
'Chart.yaml')).read()))
chart_yaml = dotify(
yaml.load(
open(os.path.join(self.source_directory, 'Chart.yaml'))
.read()))
# construct Metadata object
return Metadata(
description=chart_yaml.description,
name=chart_yaml.name,
version=chart_yaml.version
)
version=chart_yaml.version)
def get_files(self):
'''
@ -127,8 +128,8 @@ class ChartBuilder(object):
# create config object representing unmarshaled values.yaml
if os.path.exists(os.path.join(self.source_directory, 'values.yaml')):
raw_values = open(os.path.join(self.source_directory,
'values.yaml')).read()
raw_values = open(
os.path.join(self.source_directory, 'values.yaml')).read()
else:
LOG.warn("No values.yaml in %s, using empty values",
self.source_directory)
@ -144,24 +145,25 @@ class ChartBuilder(object):
# process all files in templates/ as a template to attach to the chart
# building a Template object
templates = []
if not os.path.exists(os.path.join(self.source_directory,
'templates')):
if not os.path.exists(
os.path.join(self.source_directory, 'templates')):
LOG.warn("Chart %s has no templates directory. "
"No templates will be deployed", self.chart.name)
for root, _, files in os.walk(os.path.join(self.source_directory,
'templates'), topdown=True):
"No templates will be deployed", self.chart.chart_name)
for root, _, files in os.walk(
os.path.join(self.source_directory, 'templates'),
topdown=True):
for tpl_file in files:
tname = os.path.relpath(os.path.join(root, tpl_file),
os.path.join(self.source_directory,
'templates'))
tname = os.path.relpath(
os.path.join(root, tpl_file),
os.path.join(self.source_directory, 'templates'))
if self.ignore_file(tname):
LOG.debug('Ignoring file %s', tname)
continue
templates.append(Template(name=tname,
data=open(os.path.join(root,
tpl_file),
'r').read()))
templates.append(
Template(
name=tname,
data=open(os.path.join(root, tpl_file), 'r').read()))
return templates
def get_helm_chart(self):
@ -175,18 +177,17 @@ class ChartBuilder(object):
# [process_chart(x, is_dependency=True) for x in chart.dependencies]
dependencies = []
for chart in self.chart.dependencies:
LOG.info("Building dependency chart %s for release %s", chart.name,
self.chart.release_name)
dependencies.append(ChartBuilder(chart).get_helm_chart())
for dep in self.chart.dependencies:
LOG.info("Building dependency chart %s for release %s",
self.chart.chart_name, self.chart.release)
dependencies.append(ChartBuilder(dep.chart).get_helm_chart())
helm_chart = Chart(
metadata=self.get_metadata(),
templates=self.get_templates(),
dependencies=dependencies,
values=self.get_values(),
files=self.get_files(),
)
files=self.get_files(), )
self._helm_chart = helm_chart
return helm_chart

115
armada/handlers/manifest.py Normal file
View File

@ -0,0 +1,115 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..const import DOCUMENT_CHART, DOCUMENT_GROUP, DOCUMENT_MANIFEST
class Manifest(object):
def __init__(self, documents):
self.config = None
self.documents = documents
self.charts = []
self.groups = []
self.manifest = None
self.get_documents()
def get_documents(self):
for document in self.documents:
if document.get('schema') == DOCUMENT_CHART:
self.charts.append(document)
if document.get('schema') == DOCUMENT_GROUP:
self.groups.append(document)
if document.get('schema') == DOCUMENT_MANIFEST:
self.manifest = document
def find_chart_document(self, name):
try:
for chart in self.charts:
if chart.get('metadata').get('name') == name:
return chart
except Exception:
raise Exception(
"Could not find {} in {}".format(name, DOCUMENT_CHART))
def find_chart_group_document(self, name):
try:
for group in self.groups:
if group.get('metadata').get('name') == name:
return group
except Exception:
raise Exception(
"Could not find {} in {}".format(name, DOCUMENT_GROUP))
def build_charts_deps(self):
for chart in self.charts:
self.build_chart_deps(chart)
def build_chart_groups(self):
for chart_group in self.groups:
self.build_chart_group(chart_group)
def build_chart_deps(self, chart):
try:
dep = None
for iter, dep in enumerate(chart.get('data').get('dependencies')):
if isinstance(dep, dict):
continue
chart_dep = self.find_chart_document(dep)
self.build_chart_deps(chart_dep)
chart['data']['dependencies'][iter] = {
'chart': chart_dep.get('data')
}
except Exception:
raise Exception(
"Could not find dependency chart {} in {}".format(
dep, DOCUMENT_CHART))
def build_chart_group(self, chart_group):
try:
chart = None
for iter, chart in enumerate(chart_group.get('data').get(
'chart_group', [])):
if isinstance(chart, dict):
continue
chart_dep = self.find_chart_document(chart)
chart_group['data']['chart_group'][iter] = {
'chart': chart_dep.get('data')
}
except Exception:
raise Exception(
"Could not find chart {} in {}".format(
chart, DOCUMENT_GROUP))
def build_armada_manifest(self):
try:
group = None
for iter, group in enumerate(self.manifest.get('data').get(
'chart_groups', [])):
if isinstance(group, dict):
continue
chart_grp = self.find_chart_group_document(group)
self.manifest['data']['chart_groups'][iter] = chart_grp.get(
'data')
except Exception:
raise Exception(
"Could not find chart group {} in {}".format(
group, DOCUMENT_MANIFEST))
def get_manifest(self):
self.build_charts_deps()
self.build_chart_groups()
self.build_armada_manifest()
return {
'armada': self.manifest.get('data')
}

View File

@ -12,11 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from armada.api import server
from falcon import testing
import json
import mock
import unittest
from falcon import testing
from armada.api import server
class APITestCase(testing.TestCase):
def setUp(self):
@ -25,6 +27,7 @@ class APITestCase(testing.TestCase):
self.app = server.create(middleware=False)
class TestAPI(APITestCase):
@unittest.skip('this is incorrectly tested')
@mock.patch('armada.api.armada_controller.Handler')
def test_armada_apply(self, mock_armada):
'''

View File

@ -7,42 +7,67 @@ from armada.conf import default
default.register_opts()
from armada.handlers.armada import Armada
from armada.handlers.manifest import Manifest
class ArmadaTestCase(unittest.TestCase):
test_yaml = """
armada:
release_prefix: armada
charts:
- description: this is a test
sequenced: False
chart_group:
- chart:
name: test_chart_1
release_name: test_chart_1
namespace: test
values: {}
source:
type: git
location: git://github.com/dummy/armada
subpath: chart_1
reference: master
dependencies: []
timeout: 50
- chart:
name: test_chart_2
release_name: test_chart_2
namespace: test
values: {}
source:
type: local
location: /tmp/dummy/armada
subpath: chart_2
reference: null
dependencies: []
timeout: 5
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: example-manifest
data:
release_prefix: armada
chart_groups:
- example-group
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: example-group
data:
description: this is a test
sequenced: False
chart_group:
- example-chart-1
- example-chart-2
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: example-chart-2
data:
name: test_chart_2
release_name: test_chart_2
namespace: test
values: {}
source:
type: local
location: /tmp/dummy/armada
subpath: chart_2
reference: null
dependencies: []
timeout: 5
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: example-chart-1
data:
name: test_chart_1
release_name: test_chart_1
namespace: test
values: {}
source:
type: git
location: git://github.com/dummy/armada
subpath: chart_1
reference: master
dependencies: []
timeout: 50
"""
@unittest.skip('temp')
@mock.patch('armada.handlers.armada.git')
@mock.patch('armada.handlers.armada.lint')
@mock.patch('armada.handlers.armada.Tiller')
@ -50,7 +75,8 @@ class ArmadaTestCase(unittest.TestCase):
'''Test pre-flight checks and operations'''
armada = Armada('')
armada.tiller = mock_tiller
armada.config = yaml.load(self.test_yaml)
armada.documents = yaml.safe_load_all(self.test_yaml)
armada.config = Manifest(armada.documents).get_manifest()
CHART_SOURCES = [('git://github.com/dummy/armada', 'chart_1'),
('/tmp/dummy/armada', 'chart_2')]
@ -71,6 +97,7 @@ class ArmadaTestCase(unittest.TestCase):
self.assertEqual(chart.get('chart').get('source_dir')[1],
CHART_SOURCES[counter][1])
@unittest.skip('temp')
@mock.patch('armada.handlers.armada.git')
@mock.patch('armada.handlers.armada.lint')
@mock.patch('armada.handlers.armada.Tiller')
@ -78,7 +105,8 @@ class ArmadaTestCase(unittest.TestCase):
'''Test post-flight operations'''
armada = Armada('')
armada.tiller = mock_tiller
armada.config = yaml.load(self.test_yaml)
tmp_doc = yaml.safe_load_all(self.test_yaml)
armada.config = Manifest(tmp_doc).get_manifest()
CHART_SOURCES = [('git://github.com/dummy/armada', 'chart_1'),
('/tmp/dummy/armada', 'chart_2')]
@ -97,6 +125,7 @@ class ArmadaTestCase(unittest.TestCase):
mock_git.source_cleanup \
.assert_called_with(CHART_SOURCES[counter][0])
@unittest.skip('temp')
@mock.patch.object(Armada, 'post_flight_ops')
@mock.patch.object(Armada, 'pre_flight_ops')
@mock.patch('armada.handlers.armada.ChartBuilder')
@ -110,7 +139,8 @@ class ArmadaTestCase(unittest.TestCase):
wait=True,
timeout=1000)
armada.tiller = mock_tiller
armada.config = yaml.load(self.test_yaml)
tmp_doc = yaml.safe_load_all(self.test_yaml)
armada.config = Manifest(tmp_doc).get_manifest()
charts = armada.config['armada']['charts'][0]['chart_group']
chart_1 = charts[0]['chart']

View File

@ -0,0 +1,56 @@
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: example-manifest
data:
release_prefix: example
chart_groups:
- example-group
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: example-group
data:
description: "OpenStack Infra Services"
chart_group:
- example-chart
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: example-chart
data:
name: keystone
release: keystone
namespace: undercloud
timeout: 100
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/example/example
subpath: example-chart
reference: master
dependencies:
- dep-chart
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: dep-chart
data:
name: dep-chart
release: null
namespace: null
values: {}
source:
type: git
location: git://github.com/example/example
subpath: dep-chart
reference: master
dependencies: []

View File

@ -14,66 +14,140 @@
import unittest
import yaml
import os
from armada.utils import lint
class LintTestCase(unittest.TestCase):
def setUp(self):
self.basepath = os.path.join(os.path.dirname(__file__))
def test_lint_armada_yaml_pass(self):
config = yaml.load("""
armada:
release_prefix: armada-test
charts:
- chart_group:
- chart:
name: chart
release_name: chart
namespace: chart
""")
resp = lint.valid_manifest(config)
template = '{}/templates/valid_armada_document.yaml'.format(
self.basepath)
document = yaml.safe_load_all(open(template).read())
resp = lint.validate_armada_documents(document)
self.assertTrue(resp)
def test_lint_armada_keyword_removed(self):
config = yaml.load("""
armasda:
release_prefix: armada-test
charts:
- chart_group:
- chart:
name: chart
release_name: chart
namespace: chart
""")
def test_lint_armada_manifest_no_groups(self):
template_manifest = """
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: example-manifest
data:
release_prefix: example
"""
document = yaml.safe_load_all(template_manifest)
with self.assertRaises(Exception):
lint.valid_manifest(config)
lint.validate_armada_documents(document)
def test_lint_prefix_keyword_removed(self):
config = yaml.load("""
armada:
release: armada-test
charts:
- chart_group:
- chart:
name: chart
release_name: chart
namespace: chart
""")
def test_lint_validate_manifest_pass(self):
template_manifest = """
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: example-manifest
data:
release_prefix: example
chart_groups:
- example-group
"""
document = yaml.safe_load_all(template_manifest)
self.assertTrue(lint.validate_manifest_document(document))
def test_lint_validate_manifest_no_prefix(self):
template_manifest = """
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: example-manifest
data:
chart_groups:
- example-group
"""
document = yaml.safe_load_all(template_manifest)
with self.assertRaises(Exception):
lint.valid_manifest(config)
lint.validate_manifest_document(document)
def test_lint_armada_removed(self):
config = yaml.load("""
sarmada:
release_prefix: armada-test
charts:
- chart_group:
- chart:
name: chart
release_name: chart
namespace: chart
""")
def test_lint_validate_group_pass(self):
template_manifest = """
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: example-manifest
data:
description: this is sample
chart_group:
- example-group
"""
document = yaml.safe_load_all(template_manifest)
self.assertTrue(lint.validate_chart_group_document(document))
def test_lint_validate_group_no_chart_group(self):
template_manifest = """
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: example-manifest
data:
description: this is sample
"""
document = yaml.safe_load_all(template_manifest)
with self.assertRaises(Exception):
lint.valid_manifest(config)
lint.validate_chart_group_document(document)
def test_lint_validate_chart_pass(self):
template_manifest = """
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: example-chart
data:
name: keystone
release: keystone
namespace: undercloud
timeout: 100
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/example/example
subpath: example-chart
reference: master
dependencies:
- dep-chart
"""
document = yaml.safe_load_all(template_manifest)
self.assertTrue(lint.validate_chart_document(document))
def test_lint_validate_chart_no_release(self):
template_manifest = """
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: example-chart
data:
name: keystone
namespace: undercloud
timeout: 100
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/example/example
subpath: example-chart
reference: master
dependencies:
- dep-chart
"""
document = yaml.safe_load_all(template_manifest)
with self.assertRaises(Exception):
lint.validate_chart_document(document)

View File

@ -12,26 +12,87 @@
# See the License for the specific language governing permissions and
# limitations under the License.
ARMADA_DEFINITION = 'armada'
RELEASE_PREFIX = 'release_prefix'
CHARTS_DEFINITION = 'charts'
from armada.const import DOCUMENT_CHART, DOCUMENT_GROUP, DOCUMENT_MANIFEST
from armada.const import KEYWORD_ARMADA, KEYWORD_PREFIX, KEYWORD_GROUPS, \
KEYWORD_CHARTS, KEYWORD_RELEASE
def validate_armada_documents(documents):
manifest = validate_manifest_document(documents)
group = validate_chart_group_document(documents)
chart = validate_chart_document(documents)
def valid_manifest(config):
if not isinstance(config.get(ARMADA_DEFINITION, None), dict):
raise Exception("Did not declare armada object")
return manifest and group and chart
armada_config = config.get('armada')
def validate_armada_object(object):
if not isinstance(object.get(KEYWORD_ARMADA, None), dict):
raise Exception("Could not find {} keyword".format(KEYWORD_ARMADA))
if not isinstance(armada_config.get(RELEASE_PREFIX), basestring):
raise Exception('Release needs to be a string')
armada_object = object.get('armada')
if not isinstance(armada_config.get(CHARTS_DEFINITION), list):
raise Exception('Check yaml invalid chart definition must be array')
if not isinstance(armada_object.get(KEYWORD_PREFIX), str):
raise Exception("Could not find {} keyword".format(KEYWORD_PREFIX))
for group in armada_config.get('charts'):
for chart in group.get('chart_group'):
if not isinstance(chart.get('chart').get('name'), basestring):
raise Exception('Chart name needs to be a string')
if not isinstance(armada_object.get(KEYWORD_GROUPS), list):
raise Exception(
'{} is of correct type: {} (expected: {} )'.format(
KEYWORD_GROUPS,
type(armada_object.get(KEYWORD_GROUPS)), list))
for group in armada_object.get(KEYWORD_GROUPS):
for chart in group.get(KEYWORD_CHARTS):
chart_obj = chart.get('chart')
if not isinstance(chart_obj.get(KEYWORD_RELEASE), str):
raise Exception(
'Could not find {} in {}'.format(KEYWORD_RELEASE,
chart_obj.get('name')))
return True
def validate_manifest_document(documents):
manifest_documents = []
for document in documents:
if document.get('schema') == DOCUMENT_MANIFEST:
manifest_documents.append(document)
manifest_data = document.get('data')
if not manifest_data.get(KEYWORD_PREFIX, False):
raise Exception(
'Missing {} keyword in manifest'.format(KEYWORD_PREFIX))
if not isinstance(manifest_data.get('chart_groups'),
list) and not manifest_data.get(
'chart_groups', False):
raise Exception(
'Missing %s values. Expecting list type'.format(
KEYWORD_GROUPS))
if len(manifest_documents) > 1:
raise Exception(
'Schema {} must be unique'.format(DOCUMENT_MANIFEST))
return True
def validate_chart_group_document(documents):
for document in documents:
if document.get('schema') == DOCUMENT_GROUP:
manifest_data = document.get('data')
if not isinstance(manifest_data.get(KEYWORD_CHARTS),
list) and not manifest_data.get(
'chart_group', False):
raise Exception(
'Missing %s values. Expecting a list type'.format(
KEYWORD_CHARTS))
return True
def validate_chart_document(documents):
for document in documents:
if document.get('schema') == DOCUMENT_CHART:
manifest_data = document.get('data')
if not isinstance(manifest_data.get(KEYWORD_RELEASE),
str) and not manifest_data.get(
KEYWORD_RELEASE, False):
raise Exception(
'Missing %s values in %s. Expecting a string type'.format(
KEYWORD_RELEASE,
document.get('metadata').get('name')))
return True

View File

@ -3,13 +3,13 @@
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Developers Guide
================
Commands Guide
==============
.. toctree::
:maxdepth: 2
:caption: Contents:
apply.rst
validate.rst
tiller.rst
apply.rst
validate.rst
tiller.rst

View File

@ -4,7 +4,9 @@ Armada RESTful API
Armada Endpoints
-----------------
.. http:post:: /armada/apply
::
Endpoint: POST /armada/apply
:string file The yaml file to apply
:>json boolean debug Enable debug logging
@ -16,43 +18,48 @@ Armada Endpoints
:>json boolean wait
:>json float timeout
.. code-block:: json
Request:
.. sourcecode:: js
{
"file": "examples/openstack-helm.yaml",
"options": {
"debug": true,
"disable_update_pre": false,
"disable_update_post": false,
"enable_chart_cleanup": false,
"skip_pre_flight": false,
"dry_run": false,
"wait": false,
"timeout": false
}
}
{
"file": "examples/openstack-helm.yaml",
"options": {
"debug": true,
"disable_update_pre": false,
"disable_update_post": false,
"enable_chart_cleanup": false,
"skip_pre_flight": false,
"dry_run": false,
"wait": false,
"timeout": false
}
}
.. code-block:: json
Results:
.. sourcecode:: js
{
"message": "success"
}
{
"message": "success"
}
Tiller Endpoints
-----------------
.. http:get:: /tiller/releases
::
Retrieves tiller releases.
Endpoint: GET /tiller/releases
Description: Retrieves tiller releases.
.. code-block:: json
Results:
.. sourcecode:: js
{
"releases": {
"armada-memcached": "openstack",
@ -63,13 +70,17 @@ Tiller Endpoints
}
}
.. http:get:: /tiller/status
::
Endpoint: GET /tiller/status
Retrieves the status of the Tiller server.
Results:
.. sourcecode:: js
.. code-block:: json
Results:
{
"message": Tiller Server is Active

View File

@ -1,16 +1,12 @@
Armada - Making Your First Armada Yaml
======================================
Armada - Making Your First Armada Manifest
==========================================
Keywords
--------
armada/Manifest/v1
------------------
+---------------------+--------+----------------------+
| keyword | type | action |
+=====================+========+======================+
| ``armada`` | object | define an |
| | | armada |
| | | release |
+---------------------+--------+----------------------+
| ``release_prefix`` | string | tag appended to the |
| | | front of all |
| | | charts |
@ -18,64 +14,72 @@ Keywords
| | | by the |
| | | yaml in |
| | | order to |
| | | manage them |
| | | manage releses |
| | | throughout their |
| | | lifecycles |
+---------------------+--------+----------------------+
| ``charts`` | array | stores the |
| | | definitions |
| | | of all |
| | | charts |
+---------------------+--------+----------------------+
| ``chart`` | object | definition |
| | | of the |
| | | chart |
| ``chart_groups`` | array | references |
| | | ChartGroup document |
| | | of all groups |
| | | |
+---------------------+--------+----------------------+
Defining a chart
~~~~~~~~~~~~~~~~
Example
~~~~~~~~
To define your charts is not any different than helm. we do provide some
post/pre actions that will help us manage our charts better.
::
Behavior
^^^^^^^^
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: simple-armada
data:
release_prefix: armada
chart_groups:
- chart_group
1. will check if chart exists
1. if it does not exist
- we will install the chart
2. if exist then
- armada will check if there are any differences in the charts
- if the charts are different then it will execute an upgrade
- else it will not perform any actions
Chart Keywords
^^^^^^^^^^^^^^
Chart Group
^^^^^^^^^^^
armada/ChartGroup/v1
--------------------
+-----------------+----------+------------------------------------------------------------------------+
| keyword | type | action |
+=================+==========+========================================================================+
| description | string | description of chart set |
+-----------------+----------+------------------------------------------------------------------------+
| chart_group | array | stores definition of the charts in a group |
| chart_group | array | reference to chart document |
+-----------------+----------+------------------------------------------------------------------------+
| sequenced | bool | enables sequenced chart deployment in a group |
| sequenced | bool | enables sequenced chart deployment in a group |
+-----------------+----------+------------------------------------------------------------------------+
Example
~~~~~~~~
::
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: blog-group
data:
description: Deploys Simple Service
sequenced: False
chart_group:
- chart
- chart
armada/Chart/v1
---------------
Chart
^^^^^
+-----------------+----------+------------------------------------------------------------------------+
| keyword | type | action |
+=================+==========+========================================================================+
| name | string | name for the chart |
| chart\_name | string | name for the chart |
+-----------------+----------+------------------------------------------------------------------------+
| release\_name | string | name of the release |
+-----------------+----------+------------------------------------------------------------------------+
@ -109,6 +113,27 @@ Source
| reference | string | branch of the repo |
+-------------+----------+---------------------------------------------------------------+
Defining a Chart
~~~~~~~~~~~~~~~~
To define your charts is not any different than helm. we do provide some
post/pre actions that will help us manage our charts better.
Behavior
^^^^^^^^
1. will check if chart exists
1. if it does not exist
- we will install the chart
2. if exist then
- armada will check if there are any differences in the charts
- if the charts are different then it will execute an upgrade
- else it will not perform any actions
.. note::
You can use references in order to build your charts, this will reduce the size of the chart definition will show example in multichart below
@ -118,87 +143,109 @@ Simple Example
::
armada:
release_prefix: "my_armada"
charts:
- description: I am a chart group
sequenced: False
chart_group:
- chart: &cockroach
name: cockroach
release_name: cockroach
namespace: db
timeout: 20
install:
no_hooks: false
values:
Replicas: 1
source:
type: git
location: git://github.com/kubernetes/charts/
subpath: stable/cockroachdb
reference: master
dependencies: []
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release: blog-1
namespace: default
values: {}
source:
type: git
location: https://github.com/namespace/repo
subpath: .
reference: master
dependencies: []
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: blog-group
data:
description: Deploys Simple Service
sequenced: False
chart_group:
- blog-1
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: simple-armada
data:
release_prefix: armada
chart_groups:
- blog-group
Multichart Example
~~~~~~~~~~~~~~~~~~
::
armada:
release_prefix: "my_armada"
charts:
- description: I am group 1
sequenced: True
chart_group:
- chart: &common
name: common
release_name: common
namespace: db
timeout: 20
install:
no_hooks: false
values:
Replicas: 1
source:
type: git
location: git://github.com/kubernetes/charts/
subpath: stable/common
reference: master
dependencies: []
- chart: &cockroach
name: cockroach
release_name: cockroach
namespace: db
timeout: 20
install:
no_hooks: false
values:
Replicas: 1
source:
type: git
location: git://github.com/kubernetes/charts/
subpath: stable/cockroachdb
reference: master
dependencies: []
- description: I am group 2
sequenced: False
chart_group:
- chart: &mariadb
name: mariadb
release_name: mariadb
namespace: db
timeout: 20
install:
no_hooks: false
values:
Replicas: 1
source:
type: git
location: git://github.com/kubernetes/charts/
subpath: stable/mariadb
reference: master
dependencies: []
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release: blog-1
namespace: default
values: {}
source:
type: git
location: https://github.com/namespace/repo
subpath: .
reference: master
dependencies: []
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-2
data:
chart_name: blog-2
release: blog-2
namespace: default
values: {}
source:
type: git
location: https://github.com/namespace/repo
subpath: .
reference: master
dependencies: []
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: blog-group-1
data:
description: Deploys Simple Service
sequenced: False
chart_group:
- blog-2
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: blog-group-2
data:
description: Deploys Simple Service
sequenced: False
chart_group:
- blog-1
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: simple-armada
data:
release_prefix: armada
chart_groups:
- blog-group-1
- blog-group-2
References
~~~~~~~~~~

View File

@ -13,3 +13,4 @@ Operations Guide
guide-troubleshooting.rst
guide-build-armada-yaml.rst
guide-use-armada.rst
guide-api.rst

View File

@ -0,0 +1,165 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: helm-toolkit
data:
chart_name: helm-toolkit
release: null
namespace: null
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: helm-toolkit
reference: master
dependencies: []
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: mariadb
data:
chart_name: mariadb
release: mariadb
namespace: undercloud
timeout: 3600
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: mariadb
reference: master
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: memcached
data:
chart_name: memcached
release: memcached
namespace: undercloud
timeout: 100
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: memcached
reference: master
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: etcd
data:
chart_name: etcd
release: etcd
namespace: undercloud
timeout: 3600
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: etcd
reference: master
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: rabbitmq
data:
chart_name: rabbitmq
release: rabbitmq
namespace: undercloud
timeout: 100
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: rabbitmq
reference: master
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: keystone
data:
chart_name: keystone
release: keystone
namespace: undercloud
timeout: 100
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- name: keystone-db-sync
type: job
- name: keystone-db-init
type: job
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: keystone
reference: master
dependencies:
- helm-toolkit
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: openstack-infra-services
data:
description: "OpenStack Infra Services"
sequenced: True
chart_group:
- mariadb
- etcd
- rabbitmq
- memcached
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: openstack-keystone
data:
description: "Deploying OpenStack Keystone"
sequenced: False
chart_group:
- keystone
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: armada-manifest
data:
release_prefix: armada
chart_groups:
- openstack-infra-services
- openstack-keystone

View File

@ -1,132 +0,0 @@
armada:
release_prefix: armada
charts:
- description: Deploys Helm Toolkit and Mariadb
sequenced: True
chart_group:
- chart: &helm-toolkit
name: helm-toolkit
release_name: null
namespace: null
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: helm-toolkit
reference: master
dependencies: []
- chart: &mariadb
name: mariadb
release_name: mariadb
namespace: openstack
timeout: 3360
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: mariadb
reference: master
dependencies:
- *helm-toolkit
- description: Undercloud Services add-ons
sequenced: False
chart_group:
- chart: &memcached
name: memcached
release_name: memcached
namespace: openstack
timeout: 10
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: memcached
reference: master
dependencies:
- *helm-toolkit
- chart: &etcd
name: etcd
release_name: etcd
namespace: openstack
timeout: 10
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: etcd
reference: master
dependencies:
- *helm-toolkit
- chart: &rabbitmq
name: rabbitmq
release_name: rabbitmq
namespace: openstack
timeout: 10
install:
no_hooks: false
upgrade:
no_hooks: false
values:
replicas: 1
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: rabbitmq
reference: master
dependencies:
- *helm-toolkit
- description: Openstack Services
sequenced: false
chart_group:
- chart: &keystone
name: keystone
release_name: keystone
namespace: openstack
timeout: 20
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- name: keystone-db-sync
type: job
- name: keystone-db-init
type: job
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: keystone
reference: master
dependencies:
- *helm-toolkit
- chart: &horizon
name: horizon
release_name: horizon
namespace: openstack
timeout: 10
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: horizon
reference: master
dependencies:
- *helm-toolkit

View File

@ -1,29 +1,53 @@
# For simple deployments
armada:
# simple deploy
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release: blog-1
namespace: default
values: {}
source:
type: git
location: https://github.com/gardlt/hello-world-chart
subpath: .
reference: master
dependencies: []
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-2
data:
chart_name: blog-2
release: blog-2
namespace: default
values: {}
source:
type: git
location: https://github.com/gardlt/hello-world-chart
subpath: .
reference: master
dependencies: []
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: blog-group
data:
description: Deploys Simple Service
sequenced: False
chart_group:
- blog-1
- blog-2
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: simple-armada
data:
release_prefix: armada
charts:
- description: Deploys Helm Toolkit and Mariadb
chart_group:
- chart: &blog-1
name: blog-1
release_name: blog-1
namespace: default
values: {}
source:
type: git
location: https://github.com/gardlt/hello-world-chart
subpath: .
reference: master
dependencies: []
- chart: &blog-2
name: blog-2
release_name: blog-2
namespace: default
values: {}
source:
type: git
location: https://github.com/gardlt/hello-world-chart
subpath: .
reference: master
dependencies: []
chart_groups:
- blog-group

View File

@ -25,6 +25,6 @@ commands = flake8 .
commands = nosetest -w armada
[flake8] #TODO: Remove E402
ignore=E302,H306,E402
ignore=E302,H306,E402,W503
exclude= libgit2-0.25.0, .git, .idea, .tox, *.egg-info, *.eggs, bin, dist, hapi