# Copyright 2017 The Armada Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import yaml from google.protobuf.any_pb2 import Any from hapi.chart.chart_pb2 import Chart from hapi.chart.config_pb2 import Config from hapi.chart.metadata_pb2 import Metadata from hapi.chart.template_pb2 import Template from oslo_config import cfg from oslo_log import log as logging from armada.exceptions import chartbuilder_exceptions LOG = logging.getLogger(__name__) CONF = cfg.CONF class ChartBuilder(object): ''' This class handles taking chart intentions as a parameter and turning those into proper ``protoc`` Helm charts that can be pushed to Tiller. ''' def __init__(self, chart): '''Initialize the :class:`ChartBuilder` class. :param dict chart: The document containing all intentions to pass to Tiller. ''' # cache for generated protoc chart object self._helm_chart = None # store chart schema self.chart = chart # extract, pull, whatever the chart from its source self.source_directory = self.get_source_path() # load ignored files from .helmignore if present self.ignored_files = self.get_ignored_files() def get_source_path(self): '''Return the joined path of the source directory and subpath. Returns "/" taken from the "source_dir" property from the chart, or else "" if the property isn't a 2-tuple. ''' source_dir = self.chart.get('source_dir') return (os.path.join(*source_dir) if (source_dir and isinstance(source_dir, (list, tuple)) and len(source_dir) == 2) else "") def get_ignored_files(self): '''Load files to ignore from .helmignore if present.''' try: ignored_files = [] if os.path.exists( os.path.join(self.source_directory, '.helmignore')): with open(os.path.join(self.source_directory, '.helmignore')) as f: ignored_files = f.readlines() return [filename.strip() for filename in ignored_files] except Exception: raise chartbuilder_exceptions.IgnoredFilesLoadException() def ignore_file(self, filename): '''Returns whether a given ``filename`` should be ignored. :param filename: Filename to compare against list of ignored files. :returns: True if file matches an ignored file wildcard or exact name, False otherwise. ''' for ignored_file in self.ignored_files: if (ignored_file.startswith('*') and filename.endswith(ignored_file.strip('*'))): return True elif ignored_file == filename: return True return False def get_metadata(self): '''Extract metadata from Chart.yaml to construct an instance of :class:`hapi.chart.metadata_pb2.Metadata`. ''' try: with open(os.path.join(self.source_directory, 'Chart.yaml')) as f: chart_yaml = yaml.safe_load(f.read().encode('utf-8')) except Exception: raise chartbuilder_exceptions.MetadataLoadException() # Construct Metadata object. return Metadata( description=chart_yaml.get('description'), name=chart_yaml.get('name'), version=chart_yaml.get('version')) def get_files(self): ''' Return (non-template) files in this chart. Non-template files include all files *except* Chart.yaml, values.yaml, values.toml, and any file nested under charts/ or templates/. The only exception to this rule is charts/.prov The class :class:`google.protobuf.any_pb2.Any` is wrapped around each file as that is what Helm uses. For more information, see: https://github.com/kubernetes/helm/blob/fa06dd176dbbc247b40950e38c09f978efecaecc/pkg/chartutil/load.go :returns: List of non-template files. :rtype: List[:class:`google.protobuf.any_pb2.Any`] ''' files_to_ignore = ['Chart.yaml', 'values.yaml', 'values.toml'] non_template_files = [] def _append_file_to_result(root, rel_folder_path, file): abspath = os.path.abspath(os.path.join(root, file)) relpath = os.path.join(rel_folder_path, file) encodings = ('utf-8', 'latin1') unicode_errors = [] for encoding in encodings: try: with open(abspath, 'r') as f: file_contents = f.read().encode(encoding) except OSError as e: LOG.debug( 'Failed to open and read file %s in the helm ' 'chart directory.', abspath) raise chartbuilder_exceptions.FilesLoadException( file=abspath, details=e) except UnicodeError as e: LOG.debug('Attempting to read %s using encoding %s.', abspath, encoding) msg = "(encoding=%s) %s" % (encoding, str(e)) unicode_errors.append(msg) else: break if len(unicode_errors) == 2: LOG.debug( 'Failed to read file %s in the helm chart directory.' ' Ensure that it is encoded using utf-8.', abspath) raise chartbuilder_exceptions.FilesLoadException( file=abspath, clazz=unicode_errors[0].__class__.__name__, details='\n'.join(e for e in unicode_errors)) non_template_files.append( Any(type_url=relpath, value=file_contents)) for root, dirs, files in os.walk(self.source_directory): relfolder = os.path.split(root)[-1] rel_folder_path = os.path.relpath(root, self.source_directory) if not any( root.startswith(os.path.join(self.source_directory, x)) for x in ['templates', 'charts']): for file in files: if (file not in files_to_ignore and file not in non_template_files): _append_file_to_result(root, rel_folder_path, file) elif relfolder == 'charts' and '.prov' in files: _append_file_to_result(root, rel_folder_path, '.prov') return non_template_files def get_values(self): '''Return the chart's (default) values.''' # create config object representing unmarshaled values.yaml if os.path.exists(os.path.join(self.source_directory, 'values.yaml')): with open(os.path.join(self.source_directory, 'values.yaml')) as f: raw_values = f.read() else: LOG.warn("No values.yaml in %s, using empty values", self.source_directory) raw_values = '' return Config(raw=raw_values) def get_templates(self): '''Return all the chart templates. Process all files in templates/ as a template to attach to the chart, building a :class:`hapi.chart.template_pb2.Template` object. ''' chart_name = self.chart.get('chart_name') templates = [] if not os.path.exists( os.path.join(self.source_directory, 'templates')): LOG.warn( "Chart %s has no templates directory. " "No templates will be deployed", chart_name) for root, _, files in os.walk( os.path.join(self.source_directory, 'templates'), topdown=True): for tpl_file in files: tname = os.path.relpath( os.path.join(root, tpl_file), os.path.join(self.source_directory, 'templates')) if self.ignore_file(tname): LOG.debug('Ignoring file %s', tname) continue with open(os.path.join(root, tpl_file)) as f: templates.append( Template(name=tname, data=f.read().encode())) return templates def get_helm_chart(self): '''Return a Helm chart object. Constructs a :class:`hapi.chart.chart_pb2.Chart` object from the ``chart`` intentions, including all dependencies. ''' if self._helm_chart: return self._helm_chart dependencies = [] chart_dependencies = self.chart.get('dependencies', []) chart_name = self.chart.get('chart_name', None) chart_release = self.chart.get('release', None) for dep in chart_dependencies: dep_chart = dep.get('chart', {}) dep_chart_name = dep_chart.get('chart_name', None) LOG.info("Building dependency chart %s for release %s.", dep_chart_name, chart_release) try: dependencies.append(ChartBuilder(dep_chart).get_helm_chart()) except Exception: raise chartbuilder_exceptions.DependencyException(chart_name) try: helm_chart = Chart( metadata=self.get_metadata(), templates=self.get_templates(), dependencies=dependencies, values=self.get_values(), files=self.get_files()) except Exception as e: raise chartbuilder_exceptions.HelmChartBuildException( chart_name, details=e) self._helm_chart = helm_chart return helm_chart def dump(self): '''Dumps a chart object as a serialized string so that we can perform a diff. It recurses into dependencies. ''' return self.get_helm_chart().SerializeToString()