airship
/
armada
Code Issues Proposed changes

Merge pull request #50 from gardlt/feature/api/generating-v1-endpoints 

Feature/api/generating v1 endpoints
This commit is contained in:
Alexis Rivera DeLa Torre 2017-05-01 19:28:06 -05:00 committed by GitHub
parent fb26d4ea2c 629a118f49
commit 47f4f0b5f9
16 changed files with 380 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/PULL_REQUEST_TEMPLATE.md vendored
View File

@ -1,5 +1,5 @@
<!--
Thanks for contributing to OpenStack-Helm! Please be thorough
Thanks for contributing to Armada! Please be thorough
when filling out your pull request. If the purpose for your pull
request is not clear, we may close your pull request and ask you
to resubmit.

29
Dockerfile
View File

@ -1,8 +1,9 @@
FROM ubuntu:16.04
MAINTAINER bjozsa@att.com
MAINTAINER Armada Team
ENV USER=armada \
VERSION=master
VERSION=master \
REPO=https://github.com/att-comdev/armada.git
RUN apt-get update && \
apt-get install -y \
@ -15,20 +16,18 @@ RUN apt-get update && \
gcc \
libssl-dev \
libffi-dev \
libgit2-dev
libgit2-dev \
RUN useradd -ms /bin/bash $USER
USER $USER
WORKDIR /home/$USER
RUN git clone -b $VERSION $REPO ${HOME}/armada
WORKDIR /root/armada
RUN pip install -r requirements.txt \
&& sh scripts/libgit2.sh \
&& pip install --upgrade urllib3 \
&& pip install pygit2 \
&& python setup.py install
RUN git clone -b $VERSION https://github.com/att-comdev/armada.git
ENTRYPOINT ["armada"]
WORKDIR /home/$USER/armada
RUN virtualenv --no-site-packages /home/$USER/.armada && \
. /home/$USER/.armada/bin/activate
EXPOSE 8000
RUN /home/$USER/.armada/bin/pip install -r /home/$USER/armada/requirements.txt
RUN /home/$USER/.armada/bin/python ./setup.py install
ENTRYPOINT ["/home/$USER/.armada/bin/armada"]
CMD ["-h"]
CMD gunicorn server:api -b :8000

7
README.md
View File

@ -24,7 +24,12 @@ Its functionality may extend beyond helm, assisting in interacting with kubernet
To use this container, use these simple instructions:
```
docker run quay.io/attcomdev/armada:latest
docker build -t armada .
```
```
docker run -v ~/.kube/config:/root/.kube/config -v $(pwd)/examples/:/examples -v /tmp:/dev/log quay.io/attcomdev/armada:latest
```
## Development

73
armada/armada.py
View File

@ -1,9 +1,11 @@
import difflib
import yaml
from supermutes.dot import dotify
from chartbuilder import ChartBuilder
from tiller import Tiller
from logutil import LOG
import yaml
import difflib
class Armada(object):
'''
@ -11,24 +13,27 @@ class Armada(object):
workflows
'''
def __init__(self, args):
def __init__(self, config,
disable_update_pre=False,
disable_update_post=False,
enable_chart_cleanup=False,
dry_run=False):
'''
Initialize the Armada Engine and establish
a connection to Tiller
'''
self.args = args
# internalize config
self.config = yaml.load(open(args.config).read())
self.disable_update_pre = disable_update_pre
self.disable_update_post = disable_update_post
self.enable_chart_cleanup = enable_chart_cleanup
self.dry_run = dry_run
self.config = yaml.load(config)
self.tiller = Tiller()
def find_release_chart(self, known_releases, name):
'''
Find a release given a list of known_releases and a release name
'''
for chart_name, version, chart, values in known_releases:
for chart_name, _, chart, values in known_releases:
if chart_name == name:
return chart, values
@ -36,6 +41,11 @@ class Armada(object):
'''
Syncronize Helm with the Armada Config(s)
'''
def release_prefix(prefix, chart):
'''
how to attach prefix to chart
'''
return "{}-{}".format(prefix, chart)
# extract known charts on tiller right now
known_releases = self.tiller.list_charts()
@ -65,19 +75,21 @@ class Armada(object):
# determine install or upgrade by examining known releases
LOG.debug("RELEASE: %s", chart.release_name)
if chart.release_name in [x[0] for x in known_releases]:
if release_prefix(prefix, chart.release_name) in [x[0]
for x in
known_releases]:
# indicate to the end user what path we are taking
LOG.info("Upgrading release %s", chart.release_name)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
installed_chart, installed_values = self.find_release_chart(
known_releases, chart.release_name)
known_releases, release_prefix(prefix, chart.release_name))
if not self.args.disable_update_pre:
if not self.disable_update_pre:
pre_actions = getattr(chart.upgrade, 'pre', {})
if not self.args.disable_update_post:
if not self.disable_update_post:
post_actions = getattr(chart.upgrade, 'post', {})
# show delta for both the chart templates and the chart values
@ -93,32 +105,33 @@ class Armada(object):
continue
# do actual update
self.tiller.update_release(protoc_chart, self.args.dry_run,
chart.release_name, chart.namespace,
pre_actions, post_actions,
self.tiller.update_release(protoc_chart,
self.dry_run,
chart.release_name,
chart.namespace,
prefix, pre_actions,
post_actions,
disable_hooks=chart.
upgrade.no_hooks,
values=yaml.safe_dump(values))
# process install
else:
try:
LOG.info("Installing release %s", chart.release_name)
self.tiller.install_release(protoc_chart,
self.args.dry_run,
chart.release_name,
chart.namespace,
values=yaml.safe_dump(values))
except Exception:
LOG.error("Install failed, continuing.")
LOG.info("Installing release %s", chart.release_name)
self.tiller.install_release(protoc_chart,
self.dry_run,
chart.release_name,
chart.namespace,
prefix,
values=yaml.safe_dump(values))
LOG.debug("Cleaning up chart source in %s",
chartbuilder.source_directory)
chartbuilder.source_cleanup()
if self.args.enable_chart_cleanup:
self.tiller.chart_cleanup(prefix, self.config['armada']['charts'],
known_releases)
if self.enable_chart_cleanup:
self.tiller.chart_cleanup(prefix, self.config['armada']['charts'])
def show_diff(self, chart, installed_chart,
installed_values, target_chart, target_values):
@ -146,4 +159,4 @@ class Armada(object):
for line in values_diff:
LOG.debug(line)
return (len(chart_diff) > 0) or (len(chart_diff) > 0)
return (len(chart_diff) > 0) or (len(values_diff) > 0)

26
armada/chartbuilder.py
View File

@ -1,14 +1,15 @@
import os
import yaml
from hapi.chart.template_pb2 import Template
from hapi.chart.chart_pb2 import Chart
from hapi.chart.metadata_pb2 import Metadata
from hapi.chart.config_pb2 import Config
from logutil import LOG
from supermutes.dot import dotify
import shutil
import tempfile
import pygit2
import os
import yaml
from logutil import LOG
from git import git_clone, source_cleanup
class ChartBuilder(object):
'''
@ -50,9 +51,6 @@ class ChartBuilder(object):
'''
if self.chart.source.type == 'git':
tmpdir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
self._source_tmp_dir = tmpdir
if self.parent:
LOG.info("Cloning %s/%s as dependency for %s",
self.chart.source.location,
@ -64,11 +62,11 @@ class ChartBuilder(object):
self.chart.source.subpath,
self.chart.release_name)
pygit2.clone_repository(self.chart.source.location, tmpdir,
checkout_branch=self.chart.
source.reference)
self._source_tmp_dir = git_clone(self.chart.source.location,
self.chart.source.reference)
return os.path.join(tmpdir, self.chart.source.subpath)
return os.path.join(self._source_tmp_dir,
self.chart.source.subpath)
else:
LOG.exception("Unknown source type %s for chart %s",
@ -79,7 +77,7 @@ class ChartBuilder(object):
'''
Cleanup source
'''
shutil.rmtree(self._source_tmp_dir)
source_cleanup(self._source_tmp_dir)
def get_metadata(self):
'''

18
armada/git.py Normal file
View File

@ -0,0 +1,18 @@
import pygit2
import tempfile
import shutil
def git_clone(repo_url, branch='master'):
'''
clones repo to a /tmp/ dir
'''
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch)
return _tmp_dir
def source_cleanup(target_dir):
'''
Clean up source
'''
shutil.rmtree(target_dir)

5
armada/k8s.py
View File

@ -1,5 +1,6 @@
from kubernetes import client, config
from kubernetes.client.rest import ApiException
from logutil import LOG
class K8s(object):
@ -40,5 +41,5 @@ class K8s(object):
This will return a list of objects req namespace
'''
res = self.client.list_namespaced_pod(namespace)
return res
return self.client.list_namespaced_pod(namespace)

4
armada/logutil.py
View File

@ -6,13 +6,13 @@ LOG = logging.getLogger('armada')
LOG_FORMAT = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s'
LOG_DATE = '%m-%d %H:%M'
def setup_logging(args):
def setup_logging(debug):
'''
:params args - logging information arguments in cli
various utilties to support logging
'''
level = logging.DEBUG if args.debug else logging.INFO
level = logging.DEBUG if debug else logging.INFO
logging.basicConfig(level=level, format=LOG_FORMAT, date_fmt=LOG_DATE)
handler = SysLogHandler(address='/dev/log')
syslog_formatter = logging.Formatter('%(name)s: %(levelname)s %(message)s')

20
armada/tiller.py
View File

@ -1,7 +1,7 @@
import grpc
from hapi.services.tiller_pb2 import ReleaseServiceStub, ListReleasesRequest, \
InstallReleaseRequest, UpdateReleaseRequest, UninstallReleaseRequest
from hapi.chart.config_pb2 import Config
import grpc
from k8s import K8s
from logutil import LOG
@ -65,6 +65,15 @@ class Tiller(object):
'''Stub method to support arbitrary ports in the future'''
return TILLER_PORT
def tiller_status(self):
'''
return if tiller exist or not
'''
if self._get_tiller_ip:
return True
return False
def list_releases(self):
'''
List Helm Releases
@ -134,7 +143,7 @@ class Tiller(object):
except Exception:
LOG.debug("POST: Could not create anything, please check yaml")
def update_release(self, chart, dry_run, name, namespace,
def update_release(self, chart, dry_run, name, namespace, prefix,
pre_actions=None, post_actions=None,
disable_hooks=False, values=None):
'''
@ -155,7 +164,7 @@ class Tiller(object):
dry_run=dry_run,
disable_hooks=disable_hooks,
values=values,
name=name)
name="{}-{}".format(prefix, name))
stub.UpdateRelease(release_request, self.timeout,
metadata=self.metadata)
@ -202,7 +211,7 @@ class Tiller(object):
self.timeout,
metadata=self.metadata)
def chart_cleanup(self, prefix, charts, known_releases):
def chart_cleanup(self, prefix, charts):
'''
:params charts - list of yaml charts
:params known_release - list of releases in tiller
@ -210,6 +219,9 @@ class Tiller(object):
:result - will remove any chart that is not present in yaml
'''
def release_prefix(prefix, chart):
'''
how to attach prefix to chart
'''
return "{}-{}".format(prefix, chart["chart"]["release_name"])
valid_charts = [release_prefix(prefix, chart) for chart in charts]

77
examples/armada.yaml
View File

@ -1,77 +0,0 @@
endpoints: &endpoints
glance:
this: is an example
armada:
# results in "armada-keystone" release name below
# to avoid manipulating releases managed directtly
# with helm or other armadas
release_prefix: armada
charts:
# silent dependency
- chart: &helm-toolkit
name: helm-toolkit
release_name: null
namespace: null
values: {}
source:
type: git
location: git://github.com/att-comdev/openstack-helm
subpath: helm-toolkit
reference: master
dependencies: []
- chart: &mariadb
name: mariadb
release_name: mariadb
namespace: openstack
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete: []
create: []
post:
delete: []
create: []
values:
endpoints: *endpoints
source:
type: git
location: git://github.com/att-comdev/openstack-helm
subpath: mariadb
reference: master
dependencies:
- *helm-toolkit
- chart: &keystone
name: keystone
release_name: keystone
namespace: openstack
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- name: keystone-db-sync
type: job
- name: keystone-db-init
type: job
create: []
post:
delete: []
create: []
values:
endpoints: *endpoints
source:
type: git
location: git://github.com/att-comdev/openstack-helm
subpath: keystone
reference: master
dependencies:
- *helm-toolkit

176
examples/openstack-helm.yaml Normal file
View File

@ -0,0 +1,176 @@
endpoints: &endpoints
hello-world:
this: is an example
armada:
# results in "armada-keystone" release name below
# to avoid manipulating releases managed directtly
# with helm or other armadas
release_prefix: armada
charts:
# silent dependency
- chart: &helm-toolkit
name: helm-toolkit
release_name: null
namespace: null
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: helm-toolkit
reference: master
dependencies: []
- chart: &mariadb
name: mariadb
release_name: mariadb
namespace: openstack
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete: []
create: []
post:
delete: []
create: []
values:
endpoints: *endpoints
replicas: 1
volume:
size: 1Gi
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: mariadb
reference: master
dependencies:
- *helm-toolkit
- chart: &memcached
name: memcached
release_name: memcached
namespace: openstack
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete: []
create: []
post:
delete: []
create: []
values:
endpoints: *endpoints
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: memcached
reference: master
dependencies:
- *helm-toolkit
- chart: &etcd
name: etcd
release_name: etcd
namespace: openstack
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete: []
create: []
post:
delete: []
create: []
values:
endpoints: *endpoints
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: etcd
reference: master
dependencies:
- *helm-toolkit
- chart: &rabbitmq
name: rabbitmq
release_name: rabbitmq
namespace: openstack
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete: []
create: []
post:
delete: []
create: []
values:
endpoints: *endpoints
replicas: 1
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: rabbitmq
reference: master
dependencies:
- *helm-toolkit
- chart: &keystone
name: keystone
release_name: keystone
namespace: openstack
install:
no_hooks: false
upgrade:
no_hooks: false
post:
delete: []
create: []
pre:
delete:
- name: keystone-db-sync
type: job
- name: keystone-db-init
type: job
values:
endpoints: *endpoints
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: keystone
reference: master
dependencies:
- *helm-toolkit
- chart: &horizon
name: horizon
release_name: horizon
namespace: openstack
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete: []
create: []
post:
delete: []
create: []
values:
endpoints: *endpoints
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: horizon
reference: master
dependencies:
- *helm-toolkit

3
requirements.txt
View File

@ -5,3 +5,6 @@ kubernetes==1.0.0
protobuf==3.2.0
PyYAML==3.12
supermutes==0.2.5
falcon==1.1.0
gunicorn==19.7.1

8
scripts/armada
View File

@ -30,14 +30,18 @@ def parse_args():
def run(args):
armada = Armada(args)
armada = Armada(open(args.config).read(),
args.disable_update_pre,
args.disable_update_post,
args.enable_chart_cleanup,
args.dry_run)
armada.sync()
if __name__ == '__main__':
args = parse_args()
setup_logging(args)
setup_logging(args.debug)
try:
run(args)

12
scripts/libgit2.sh
View File

@ -2,9 +2,9 @@
# Ubuntu 16.04 Install only
sudo apt install git cmake make -y
sudo apt-get install -y python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libssh2-1 libgit2-dev python-pip libgit2-24
sudo apt-get install -y pkg-config libssh2-1-dev libhttp-parser-dev libssl-dev libz-dev
apt install git cmake make wget -y
apt-get install -y python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libssh2-1 libgit2-dev python-pip libgit2-24
apt-get install -y pkg-config libssh2-1-dev libhttp-parser-dev libssl-dev libz-dev
LIBGIT_VERSION='0.25.0'
@ -13,6 +13,6 @@ tar xzf v${LIBGIT_VERSION}.tar.gz
cd libgit2-${LIBGIT_VERSION}/
cmake .
make
sudo make install
sudo pip install pygit2==${LIBGIT_VERSION}
sudo ldconfig
make install
pip install pygit2==${LIBGIT_VERSION}
ldconfig

18
scripts/libgit2_docker.sh Normal file
View File

@ -0,0 +1,18 @@
#!/bin/sh
# Ubuntu 16.04 Install only
apt install git cmake make -y
apt-get install -y python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libssh2-1 libgit2-dev python-pip libgit2-24
apt-get install -y pkg-config libssh2-1-dev libhttp-parser-dev libssl-dev libz-dev
LIBGIT_VERSION='0.25.0'
wget https://github.com/libgit2/libgit2/archive/v${LIBGIT_VERSION}.tar.gz
tar xzf v${LIBGIT_VERSION}.tar.gz
cd libgit2-${LIBGIT_VERSION}/
cmake .
make
make install
pip install pygit2==${LIBGIT_VERSION}
ldconfig

56
server.py Normal file
View File

@ -0,0 +1,56 @@
import falcon
import json
from falcon import HTTP_200
from armada.tiller import Tiller as tillerHandler
from armada.armada import Armada as armadaHandler
from armada.logutil import setup_logging
class Tiller(object):
'''
tiller service endpoints
'''
def on_get(self, req, resp):
'''
get tiller status
'''
message = "Tiller Server is {}"
if tillerHandler().tiller_status():
resp.data = json.dumps({'message': message.format('Active')})
else:
resp.data = json.dumps({'message': message.format('Not Present')})
resp.content_type = 'application/json'
resp.status = HTTP_200
class Armada(object):
'''
apply armada endpoint service
'''
def on_post(self, req, resp):
armada = armadaHandler(req.stream.read())
print armada.tiller.k8s.get_namespace_pod()
armada.sync()
resp.data = json.dumps({'message': 'Success'})
resp.content_type = 'application/json'
resp.status = HTTP_200
wsgi_app = api = falcon.API()
# Routing
url_routes = (
('/tiller/status', Tiller()),
('/apply', Armada()),
)
for route, service in url_routes:
api.add_route(route, service)
setup_logging(False)