286 lines
10 KiB
Python
286 lines
10 KiB
Python
# Copyright 2017 The Armada Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
import yaml
|
|
|
|
from google.protobuf.any_pb2 import Any
|
|
from hapi.chart.chart_pb2 import Chart
|
|
from hapi.chart.config_pb2 import Config
|
|
from hapi.chart.metadata_pb2 import Metadata
|
|
from hapi.chart.template_pb2 import Template
|
|
from supermutes.dot import dotify
|
|
|
|
from oslo_config import cfg
|
|
from oslo_log import log as logging
|
|
|
|
from armada.exceptions import chartbuilder_exceptions
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
CONF = cfg.CONF
|
|
|
|
|
|
class ChartBuilder(object):
|
|
'''
|
|
This class handles taking chart intentions as a parameter and
|
|
turning those into proper protoc helm charts that can be
|
|
pushed to tiller.
|
|
|
|
It also processes chart source declarations, fetching chart
|
|
source from external resources where necessary
|
|
'''
|
|
|
|
def __init__(self, chart, parent=None):
|
|
'''
|
|
Initialize the ChartBuilder class
|
|
|
|
Note that tthis will trigger a source pull as part of
|
|
initialization as its necessary in order to examine
|
|
the source service many of the calls on ChartBuilder
|
|
'''
|
|
|
|
# cache for generated protoc chart object
|
|
self._helm_chart = None
|
|
|
|
# record whether this is a dependency based chart
|
|
self.parent = parent
|
|
|
|
# store chart schema
|
|
self.chart = chart
|
|
|
|
# extract, pull, whatever the chart from its source
|
|
self.source_directory = self.get_source_path()
|
|
|
|
# load ignored files from .helmignore if present
|
|
self.ignored_files = self.get_ignored_files()
|
|
|
|
def get_source_path(self):
|
|
'''
|
|
Return the joined path of the source directory and subpath
|
|
'''
|
|
return (
|
|
os.path.join(self.chart.source_dir[0], self.chart.source_dir[1])
|
|
if (hasattr(self.chart, 'source_dir') and
|
|
isinstance(self.chart.source_dir, (list, tuple)) and
|
|
len(self.chart.source_dir) == 2)
|
|
else ""
|
|
)
|
|
|
|
def get_ignored_files(self):
|
|
'''
|
|
Load files from .helmignore if present
|
|
'''
|
|
try:
|
|
ignored_files = []
|
|
if os.path.exists(os.path.join(self.source_directory,
|
|
'.helmignore')):
|
|
with open(os.path.join(self.source_directory,
|
|
'.helmignore')) as f:
|
|
ignored_files = f.readlines()
|
|
return [filename.strip() for filename in ignored_files]
|
|
except Exception:
|
|
raise chartbuilder_exceptions.IgnoredFilesLoadException()
|
|
|
|
def ignore_file(self, filename):
|
|
'''
|
|
:params file - filename to compare against list of ignored files
|
|
|
|
Returns true if file matches an ignored file wildcard or exact name,
|
|
false otherwise
|
|
'''
|
|
for ignored_file in self.ignored_files:
|
|
if (ignored_file.startswith('*') and
|
|
filename.endswith(ignored_file.strip('*'))):
|
|
return True
|
|
elif ignored_file == filename:
|
|
return True
|
|
return False
|
|
|
|
def get_metadata(self):
|
|
'''
|
|
Process metadata
|
|
'''
|
|
# extract Chart.yaml to construct metadata
|
|
|
|
try:
|
|
with open(os.path.join(self.source_directory, 'Chart.yaml')) as f:
|
|
chart_yaml = dotify(yaml.safe_load(f.read().encode('utf-8')))
|
|
|
|
except Exception:
|
|
raise chartbuilder_exceptions.MetadataLoadException()
|
|
|
|
# construct Metadata object
|
|
return Metadata(
|
|
description=chart_yaml.description,
|
|
name=chart_yaml.name,
|
|
version=chart_yaml.version)
|
|
|
|
def get_files(self):
|
|
'''
|
|
Return (non-template) files in this chart.
|
|
|
|
Non-template files include all files *except* Chart.yaml, values.yaml,
|
|
values.toml, and any file nested under charts/ or templates/. The only
|
|
exception to this rule is charts/.prov
|
|
|
|
The class :class:`google.protobuf.any_pb2.Any` is wrapped around
|
|
each file as that is what Helm uses.
|
|
|
|
For more information, see:
|
|
https://github.com/kubernetes/helm/blob/fa06dd176dbbc247b40950e38c09f978efecaecc/pkg/chartutil/load.go
|
|
|
|
:returns: List of non-template files.
|
|
:rtype: List[:class:`google.protobuf.any_pb2.Any`]
|
|
'''
|
|
|
|
files_to_ignore = ['Chart.yaml', 'values.yaml', 'values.toml']
|
|
non_template_files = []
|
|
|
|
def _append_file_to_result(root, rel_folder_path, file):
|
|
abspath = os.path.abspath(os.path.join(root, file))
|
|
relpath = os.path.join(rel_folder_path, file)
|
|
|
|
encodings = ('utf-8', 'latin1')
|
|
unicode_errors = []
|
|
|
|
for encoding in encodings:
|
|
try:
|
|
with open(abspath, 'r') as f:
|
|
file_contents = f.read().encode(encoding)
|
|
except OSError as e:
|
|
LOG.debug('Failed to open and read file %s in the helm '
|
|
'chart directory.', abspath)
|
|
raise chartbuilder_exceptions.FilesLoadException(
|
|
file=abspath, details=e)
|
|
except UnicodeError as e:
|
|
LOG.debug('Attempting to read %s using encoding %s.',
|
|
abspath, encoding)
|
|
msg = "(encoding=%s) %s" % (encoding, str(e))
|
|
unicode_errors.append(msg)
|
|
else:
|
|
break
|
|
|
|
if len(unicode_errors) == 2:
|
|
LOG.debug('Failed to read file %s in the helm chart directory.'
|
|
' Ensure that it is encoded using utf-8.', abspath)
|
|
raise chartbuilder_exceptions.FilesLoadException(
|
|
file=abspath, clazz=unicode_errors[0].__class__.__name__,
|
|
details='\n'.join(e for e in unicode_errors))
|
|
|
|
non_template_files.append(
|
|
Any(type_url=relpath,
|
|
value=file_contents))
|
|
|
|
for root, dirs, files in os.walk(self.source_directory):
|
|
relfolder = os.path.split(root)[-1]
|
|
rel_folder_path = os.path.relpath(root, self.source_directory)
|
|
|
|
if not any(root.startswith(os.path.join(self.source_directory, x))
|
|
for x in ['templates', 'charts']):
|
|
for file in files:
|
|
if (file not in files_to_ignore and
|
|
file not in non_template_files):
|
|
_append_file_to_result(root, rel_folder_path, file)
|
|
elif relfolder == 'charts' and '.prov' in files:
|
|
_append_file_to_result(root, rel_folder_path, '.prov')
|
|
|
|
return non_template_files
|
|
|
|
def get_values(self):
|
|
'''
|
|
Return the chart (default) values
|
|
'''
|
|
|
|
# create config object representing unmarshaled values.yaml
|
|
if os.path.exists(os.path.join(self.source_directory, 'values.yaml')):
|
|
with open(os.path.join(self.source_directory, 'values.yaml')) as f:
|
|
raw_values = f.read()
|
|
else:
|
|
LOG.warn("No values.yaml in %s, using empty values",
|
|
self.source_directory)
|
|
raw_values = ''
|
|
|
|
return Config(raw=raw_values)
|
|
|
|
def get_templates(self):
|
|
'''
|
|
Return all the chart templates
|
|
'''
|
|
|
|
# process all files in templates/ as a template to attach to the chart
|
|
# building a Template object
|
|
templates = []
|
|
if not os.path.exists(
|
|
os.path.join(self.source_directory, 'templates')):
|
|
LOG.warn("Chart %s has no templates directory. "
|
|
"No templates will be deployed", self.chart.chart_name)
|
|
for root, _, files in os.walk(
|
|
os.path.join(self.source_directory, 'templates'),
|
|
topdown=True):
|
|
for tpl_file in files:
|
|
tname = os.path.relpath(
|
|
os.path.join(root, tpl_file),
|
|
os.path.join(self.source_directory, 'templates'))
|
|
if self.ignore_file(tname):
|
|
LOG.debug('Ignoring file %s', tname)
|
|
continue
|
|
|
|
with open(os.path.join(root, tpl_file)) as f:
|
|
templates.append(
|
|
Template(name=tname, data=f.read().encode()))
|
|
|
|
return templates
|
|
|
|
def get_helm_chart(self):
|
|
'''
|
|
Return a helm chart object
|
|
'''
|
|
if self._helm_chart:
|
|
return self._helm_chart
|
|
|
|
dependencies = []
|
|
for dep in self.chart.dependencies:
|
|
LOG.info("Building dependency chart %s for release %s.",
|
|
dep.chart.chart_name, self.chart.release)
|
|
try:
|
|
dependencies.append(ChartBuilder(dep.chart).get_helm_chart())
|
|
except Exception:
|
|
chart_name = self.chart.chart_name
|
|
raise chartbuilder_exceptions.DependencyException(chart_name)
|
|
|
|
try:
|
|
helm_chart = Chart(
|
|
metadata=self.get_metadata(),
|
|
templates=self.get_templates(),
|
|
dependencies=dependencies,
|
|
values=self.get_values(),
|
|
files=self.get_files())
|
|
except Exception as e:
|
|
chart_name = self.chart.chart_name
|
|
raise chartbuilder_exceptions.HelmChartBuildException(
|
|
chart_name, details=e)
|
|
|
|
self._helm_chart = helm_chart
|
|
return helm_chart
|
|
|
|
def dump(self):
|
|
'''
|
|
This method is used to dump a chart object as a
|
|
serialized string so that we can perform a diff
|
|
|
|
It should recurse into dependencies
|
|
'''
|
|
return self.get_helm_chart().SerializeToString()
|