Merge pull request #50 from gardlt/feature/api/generating-v1-endpoints

Feature/api/generating v1 endpoints

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,5 @@
 <!--
-      Thanks for contributing to OpenStack-Helm!  Please be thorough
+      Thanks for contributing to Armada!  Please be thorough
       when filling out your pull request. If the purpose for your pull
       request is not clear, we may close your pull request and ask you
       to resubmit.

Dockerfile

@@ -1,8 +1,9 @@
 FROM ubuntu:16.04
-MAINTAINER bjozsa@att.com
+MAINTAINER Armada Team
 
 ENV USER=armada \
-    VERSION=master
+    VERSION=master \
+    REPO=https://github.com/att-comdev/armada.git
 
 RUN apt-get update && \
     apt-get install -y \
@@ -15,20 +16,18 @@ RUN apt-get update && \
         gcc \
         libssl-dev \
         libffi-dev \
-        libgit2-dev
+        libgit2-dev \
 
-RUN useradd -ms /bin/bash $USER 
+RUN git clone -b $VERSION $REPO ${HOME}/armada
-USER $USER 
+WORKDIR /root/armada
-WORKDIR /home/$USER
+RUN pip install -r requirements.txt \
+    && sh scripts/libgit2.sh \
+    && pip install --upgrade urllib3 \
+    && pip install pygit2 \
+    && python setup.py install
 
-RUN git clone -b $VERSION https://github.com/att-comdev/armada.git
+ENTRYPOINT ["armada"]
 
-WORKDIR /home/$USER/armada
+EXPOSE 8000
-RUN virtualenv --no-site-packages /home/$USER/.armada && \
-    . /home/$USER/.armada/bin/activate
 
-RUN /home/$USER/.armada/bin/pip install -r /home/$USER/armada/requirements.txt
+CMD gunicorn server:api -b :8000
-RUN /home/$USER/.armada/bin/python ./setup.py install
-
-ENTRYPOINT ["/home/$USER/.armada/bin/armada"]
-CMD ["-h"]

README.md

@@ -24,7 +24,12 @@ Its functionality may extend beyond helm, assisting in interacting with kubernet
 To use this container, use these simple instructions:
 
 ```
-docker run quay.io/attcomdev/armada:latest
+docker build -t armada .
+```
+
+
+```
+docker run -v ~/.kube/config:/root/.kube/config -v $(pwd)/examples/:/examples -v /tmp:/dev/log quay.io/attcomdev/armada:latest
 ```
 
 ## Development

armada/armada.py

@@ -1,9 +1,11 @@
+import difflib
+import yaml
+
 from supermutes.dot import dotify
+
 from chartbuilder import ChartBuilder
 from tiller import Tiller
 from logutil import LOG
-import yaml
-import difflib
 
 class Armada(object):
     '''
@@ -11,24 +13,27 @@ class Armada(object):
     workflows
     '''
 
-    def __init__(self, args):
+    def __init__(self, config,
+                 disable_update_pre=False,
+                 disable_update_post=False,
+                 enable_chart_cleanup=False,
+                 dry_run=False):
         '''
         Initialize the Armada Engine and establish
         a connection to Tiller
         '''
-
+        self.disable_update_pre = disable_update_pre
-        self.args = args
+        self.disable_update_post = disable_update_post
-
+        self.enable_chart_cleanup = enable_chart_cleanup
-        # internalize config
+        self.dry_run = dry_run
-        self.config = yaml.load(open(args.config).read())
+        self.config = yaml.load(config)
-
         self.tiller = Tiller()
 
     def find_release_chart(self, known_releases, name):
         '''
         Find a release given a list of known_releases and a release name
         '''
-        for chart_name, version, chart, values in known_releases:
+        for chart_name, _, chart, values in known_releases:
             if chart_name == name:
                 return chart, values
 
@@ -36,6 +41,11 @@ class Armada(object):
         '''
         Syncronize Helm with the Armada Config(s)
         '''
+        def release_prefix(prefix, chart):
+            '''
+            how to attach prefix to chart
+            '''
+            return "{}-{}".format(prefix, chart)
 
         # extract known charts on tiller right now
         known_releases = self.tiller.list_charts()
@@ -65,19 +75,21 @@ class Armada(object):
             # determine install or upgrade by examining known releases
             LOG.debug("RELEASE: %s", chart.release_name)
 
-            if chart.release_name in [x[0] for x in known_releases]:
+            if release_prefix(prefix, chart.release_name) in [x[0]
+                                                              for x in
+                                                              known_releases]:
 
                 # indicate to the end user what path we are taking
                 LOG.info("Upgrading release %s", chart.release_name)
                 # extract the installed chart and installed values from the
                 # latest release so we can compare to the intended state
                 installed_chart, installed_values = self.find_release_chart(
-                    known_releases, chart.release_name)
+                    known_releases, release_prefix(prefix, chart.release_name))
 
-                if not self.args.disable_update_pre:
+                if not self.disable_update_pre:
                     pre_actions = getattr(chart.upgrade, 'pre', {})
 
-                if not self.args.disable_update_post:
+                if not self.disable_update_post:
                     post_actions = getattr(chart.upgrade, 'post', {})
 
                 # show delta for both the chart templates and the chart values
@@ -93,32 +105,33 @@ class Armada(object):
                     continue
 
                 # do actual update
-                self.tiller.update_release(protoc_chart, self.args.dry_run,
+                self.tiller.update_release(protoc_chart,
-                                           chart.release_name, chart.namespace,
+                                           self.dry_run,
-                                           pre_actions, post_actions,
+                                           chart.release_name,
+                                           chart.namespace,
+                                           prefix, pre_actions,
+                                           post_actions,
                                            disable_hooks=chart.
                                            upgrade.no_hooks,
                                            values=yaml.safe_dump(values))
 
             # process install
             else:
-                try:
+                LOG.info("Installing release %s", chart.release_name)
-                    LOG.info("Installing release %s", chart.release_name)
+                self.tiller.install_release(protoc_chart,
-                    self.tiller.install_release(protoc_chart,
+                                            self.dry_run,
-                                                self.args.dry_run,
+                                            chart.release_name,
-                                                chart.release_name,
+                                            chart.namespace,
-                                                chart.namespace,
+                                            prefix,
-                                                values=yaml.safe_dump(values))
+                                            values=yaml.safe_dump(values))
-                except Exception:
-                    LOG.error("Install failed, continuing.")
 
             LOG.debug("Cleaning up chart source in %s",
                       chartbuilder.source_directory)
+
             chartbuilder.source_cleanup()
 
-        if self.args.enable_chart_cleanup:
+        if self.enable_chart_cleanup:
-            self.tiller.chart_cleanup(prefix, self.config['armada']['charts'],
+            self.tiller.chart_cleanup(prefix, self.config['armada']['charts'])
-                                      known_releases)
 
     def show_diff(self, chart, installed_chart,
                   installed_values, target_chart, target_values):
@@ -146,4 +159,4 @@ class Armada(object):
             for line in values_diff:
                 LOG.debug(line)
 
-        return (len(chart_diff) > 0) or (len(chart_diff) > 0)
+        return (len(chart_diff) > 0) or (len(values_diff) > 0)

armada/chartbuilder.py

@@ -1,14 +1,15 @@
+import os
+import yaml
+
 from hapi.chart.template_pb2 import Template
 from hapi.chart.chart_pb2 import Chart
 from hapi.chart.metadata_pb2 import Metadata
 from hapi.chart.config_pb2 import Config
-from logutil import LOG
 from supermutes.dot import dotify
-import shutil
+
-import tempfile
+from logutil import LOG
-import pygit2
+from git import git_clone, source_cleanup
-import os
+
-import yaml
 
 class ChartBuilder(object):
     '''
@@ -50,9 +51,6 @@ class ChartBuilder(object):
         '''
 
         if self.chart.source.type == 'git':
-
-            tmpdir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
-            self._source_tmp_dir = tmpdir
             if self.parent:
                 LOG.info("Cloning %s/%s as dependency for %s",
                          self.chart.source.location,
@@ -64,11 +62,11 @@ class ChartBuilder(object):
                          self.chart.source.subpath,
                          self.chart.release_name)
 
-            pygit2.clone_repository(self.chart.source.location, tmpdir,
+            self._source_tmp_dir = git_clone(self.chart.source.location,
-                                    checkout_branch=self.chart.
+                                             self.chart.source.reference)
-                                    source.reference)
 
-            return os.path.join(tmpdir, self.chart.source.subpath)
+            return os.path.join(self._source_tmp_dir,
+                                self.chart.source.subpath)
 
         else:
             LOG.exception("Unknown source type %s for chart %s",
@@ -79,7 +77,7 @@ class ChartBuilder(object):
         '''
         Cleanup source
         '''
-        shutil.rmtree(self._source_tmp_dir)
+        source_cleanup(self._source_tmp_dir)
 
     def get_metadata(self):
         '''

armada/git.py

@@ -0,0 +1,18 @@
+import pygit2
+import tempfile
+import shutil
+
+def git_clone(repo_url, branch='master'):
+    '''
+    clones repo to a /tmp/ dir
+    '''
+    _tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
+    pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch)
+
+    return _tmp_dir
+
+def source_cleanup(target_dir):
+    '''
+    Clean up source
+    '''
+    shutil.rmtree(target_dir)

armada/k8s.py

@@ -1,5 +1,6 @@
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
+
 from logutil import LOG
 
 class K8s(object):
@@ -40,5 +41,5 @@ class K8s(object):
 
         This will return a list of objects req namespace
         '''
-        res = self.client.list_namespaced_pod(namespace)
+
-        return res
+        return self.client.list_namespaced_pod(namespace)

armada/logutil.py

@@ -6,13 +6,13 @@ LOG = logging.getLogger('armada')
 LOG_FORMAT = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s'
 LOG_DATE = '%m-%d %H:%M'
 
-def setup_logging(args):
+def setup_logging(debug):
     '''
     :params args - logging information arguments in cli
 
     various utilties to support logging
     '''
-    level = logging.DEBUG if args.debug else logging.INFO
+    level = logging.DEBUG if debug else logging.INFO
     logging.basicConfig(level=level, format=LOG_FORMAT, date_fmt=LOG_DATE)
     handler = SysLogHandler(address='/dev/log')
     syslog_formatter = logging.Formatter('%(name)s: %(levelname)s %(message)s')

armada/tiller.py

@@ -1,7 +1,7 @@
+import grpc
 from hapi.services.tiller_pb2 import ReleaseServiceStub, ListReleasesRequest, \
     InstallReleaseRequest, UpdateReleaseRequest, UninstallReleaseRequest
 from hapi.chart.config_pb2 import Config
-import grpc
 
 from k8s import K8s
 from logutil import LOG
@@ -65,6 +65,15 @@ class Tiller(object):
         '''Stub method to support arbitrary ports in the future'''
         return TILLER_PORT
 
+    def tiller_status(self):
+        '''
+        return if tiller exist or not
+        '''
+        if self._get_tiller_ip:
+            return True
+
+        return False
+
     def list_releases(self):
         '''
         List Helm Releases
@@ -134,7 +143,7 @@ class Tiller(object):
         except Exception:
             LOG.debug("POST: Could not create anything, please check yaml")
 
-    def update_release(self, chart, dry_run, name, namespace,
+    def update_release(self, chart, dry_run, name, namespace, prefix,
                        pre_actions=None, post_actions=None,
                        disable_hooks=False, values=None):
         '''
@@ -155,7 +164,7 @@ class Tiller(object):
             dry_run=dry_run,
             disable_hooks=disable_hooks,
             values=values,
-            name=name)
+            name="{}-{}".format(prefix, name))
 
         stub.UpdateRelease(release_request, self.timeout,
                            metadata=self.metadata)
@@ -202,7 +211,7 @@ class Tiller(object):
                                      self.timeout,
                                      metadata=self.metadata)
 
-    def chart_cleanup(self, prefix, charts, known_releases):
+    def chart_cleanup(self, prefix, charts):
         '''
         :params charts - list of yaml charts
         :params known_release - list of releases in tiller
@@ -210,6 +219,9 @@ class Tiller(object):
         :result - will remove any chart that is not present in yaml
         '''
         def release_prefix(prefix, chart):
+            '''
+            how to attach prefix to chart
+            '''
             return "{}-{}".format(prefix, chart["chart"]["release_name"])
 
         valid_charts = [release_prefix(prefix, chart) for chart in charts]

examples/armada.yaml

@@ -1,77 +0,0 @@
-endpoints: &endpoints
-  glance:
-    this: is an example
-
-armada:
-
-  # results in "armada-keystone" release name below
-  # to avoid manipulating releases managed directtly
-  # with helm or other armadas
-  release_prefix: armada
-
-  charts:
-
-    # silent dependency
-    - chart: &helm-toolkit
-        name: helm-toolkit
-        release_name: null
-        namespace: null
-        values: {}
-        source:
-          type: git
-          location: git://github.com/att-comdev/openstack-helm
-          subpath: helm-toolkit
-          reference: master
-        dependencies: []
-
-    - chart: &mariadb
-        name: mariadb
-        release_name: mariadb
-        namespace: openstack
-        install:
-          no_hooks: false
-        upgrade:
-          no_hooks: false
-          pre:
-              delete: []
-              create: []
-          post:
-              delete: []
-              create: []
-        values:
-          endpoints: *endpoints
-        source:
-          type: git
-          location: git://github.com/att-comdev/openstack-helm
-          subpath: mariadb
-          reference: master
-        dependencies:
-          - *helm-toolkit
-
-    - chart: &keystone
-        name: keystone
-        release_name: keystone
-        namespace: openstack
-        install:
-          no_hooks: false
-        upgrade:
-          no_hooks: false
-          pre:
-              delete:
-                  - name: keystone-db-sync
-                    type: job
-                  - name: keystone-db-init
-                    type: job
-              create: []
-          post:
-              delete: []
-              create: []
-        values:
-          endpoints: *endpoints
-        source:
-          type: git
-          location: git://github.com/att-comdev/openstack-helm
-          subpath: keystone
-          reference: master
-        dependencies:
-          - *helm-toolkit

examples/openstack-helm.yaml

@@ -0,0 +1,176 @@
+endpoints: &endpoints
+  hello-world:
+    this: is an example
+
+armada:
+
+  # results in "armada-keystone" release name below
+  # to avoid manipulating releases managed directtly
+  # with helm or other armadas
+  release_prefix: armada
+
+  charts:
+
+    # silent dependency
+    - chart: &helm-toolkit
+        name: helm-toolkit
+        release_name: null
+        namespace: null
+        values: {}
+        source:
+          type: git
+          location: git://github.com/openstack/openstack-helm
+          subpath: helm-toolkit
+          reference: master
+        dependencies: []
+
+    - chart: &mariadb
+        name: mariadb
+        release_name: mariadb
+        namespace: openstack
+        install:
+          no_hooks: false
+        upgrade:
+          no_hooks: false
+          pre:
+              delete: []
+              create: []
+          post:
+              delete: []
+              create: []
+        values:
+            endpoints: *endpoints
+            replicas: 1
+            volume:
+                size: 1Gi
+        source:
+          type: git
+          location: git://github.com/openstack/openstack-helm
+          subpath: mariadb
+          reference: master
+        dependencies:
+          - *helm-toolkit
+
+    - chart: &memcached
+        name: memcached
+        release_name: memcached
+        namespace: openstack
+        install:
+          no_hooks: false
+        upgrade:
+          no_hooks: false
+          pre:
+              delete: []
+              create: []
+          post:
+              delete: []
+              create: []
+        values:
+            endpoints: *endpoints
+        source:
+          type: git
+          location: git://github.com/openstack/openstack-helm
+          subpath: memcached
+          reference: master
+        dependencies:
+          - *helm-toolkit
+
+    - chart: &etcd
+        name: etcd
+        release_name: etcd
+        namespace: openstack
+        install:
+          no_hooks: false
+        upgrade:
+          no_hooks: false
+          pre:
+              delete: []
+              create: []
+          post:
+              delete: []
+              create: []
+        values:
+            endpoints: *endpoints
+        source:
+          type: git
+          location: git://github.com/openstack/openstack-helm
+          subpath: etcd
+          reference: master
+        dependencies:
+          - *helm-toolkit
+
+    - chart: &rabbitmq
+        name: rabbitmq
+        release_name: rabbitmq
+        namespace: openstack
+        install:
+          no_hooks: false
+        upgrade:
+          no_hooks: false
+          pre:
+              delete: []
+              create: []
+          post:
+              delete: []
+              create: []
+        values:
+            endpoints: *endpoints
+            replicas: 1
+        source:
+          type: git
+          location: git://github.com/openstack/openstack-helm
+          subpath: rabbitmq
+          reference: master
+        dependencies:
+          - *helm-toolkit
+
+    - chart: &keystone
+        name: keystone
+        release_name: keystone
+        namespace: openstack
+        install:
+          no_hooks: false
+        upgrade:
+          no_hooks: false
+          post:
+              delete: []
+              create: []
+          pre:
+              delete:
+                  - name: keystone-db-sync
+                    type: job
+                  - name: keystone-db-init
+                    type: job
+        values:
+            endpoints: *endpoints
+        source:
+          type: git
+          location: git://github.com/openstack/openstack-helm
+          subpath: keystone
+          reference: master
+        dependencies:
+          - *helm-toolkit
+
+    - chart: &horizon
+        name: horizon
+        release_name: horizon
+        namespace: openstack
+        install:
+          no_hooks: false
+        upgrade:
+          no_hooks: false
+          pre:
+              delete: []
+              create: []
+          post:
+              delete: []
+              create: []
+        values:
+            endpoints: *endpoints
+        source:
+          type: git
+          location: git://github.com/openstack/openstack-helm
+          subpath: horizon
+          reference: master
+        dependencies:
+          - *helm-toolkit

requirements.txt

@@ -5,3 +5,6 @@ kubernetes==1.0.0
 protobuf==3.2.0
 PyYAML==3.12
 supermutes==0.2.5
+
+falcon==1.1.0
+gunicorn==19.7.1

scripts/armada

@@ -30,14 +30,18 @@ def parse_args():
 
 def run(args):
 
-    armada = Armada(args)
+    armada = Armada(open(args.config).read(),
+                    args.disable_update_pre,
+                    args.disable_update_post,
+                    args.enable_chart_cleanup,
+                    args.dry_run)
     armada.sync()
 
 
 if __name__ == '__main__':
 
     args = parse_args()
-    setup_logging(args)
+    setup_logging(args.debug)
 
     try:
         run(args)

scripts/libgit2.sh

@@ -2,9 +2,9 @@
 
 # Ubuntu 16.04 Install only
 
-sudo apt install git cmake make -y
+apt install git cmake make wget -y
-sudo apt-get install -y python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libssh2-1 libgit2-dev python-pip libgit2-24
+apt-get install -y python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libssh2-1 libgit2-dev python-pip libgit2-24
-sudo apt-get install -y pkg-config libssh2-1-dev libhttp-parser-dev libssl-dev libz-dev
+apt-get install -y pkg-config libssh2-1-dev libhttp-parser-dev libssl-dev libz-dev
 
 LIBGIT_VERSION='0.25.0'
 
@@ -13,6 +13,6 @@ tar xzf v${LIBGIT_VERSION}.tar.gz
 cd libgit2-${LIBGIT_VERSION}/
 cmake .
 make
-sudo make install
+make install
-sudo pip install pygit2==${LIBGIT_VERSION}
+pip install pygit2==${LIBGIT_VERSION}
-sudo ldconfig
+ldconfig

scripts/libgit2_docker.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Ubuntu 16.04 Install only
+
+apt install git cmake make -y
+apt-get install -y python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libssh2-1 libgit2-dev python-pip libgit2-24
+apt-get install -y pkg-config libssh2-1-dev libhttp-parser-dev libssl-dev libz-dev
+
+LIBGIT_VERSION='0.25.0'
+
+wget https://github.com/libgit2/libgit2/archive/v${LIBGIT_VERSION}.tar.gz
+tar xzf v${LIBGIT_VERSION}.tar.gz
+cd libgit2-${LIBGIT_VERSION}/
+cmake .
+make
+make install
+pip install pygit2==${LIBGIT_VERSION}
+ldconfig

server.py

@@ -0,0 +1,56 @@
+
+import falcon
+import json
+from falcon import HTTP_200
+
+from armada.tiller import Tiller as tillerHandler
+from armada.armada import Armada as armadaHandler
+from armada.logutil import setup_logging
+
+class Tiller(object):
+    '''
+    tiller service endpoints
+    '''
+
+    def on_get(self, req, resp):
+        '''
+        get tiller status
+        '''
+        message = "Tiller Server is {}"
+        if tillerHandler().tiller_status():
+            resp.data = json.dumps({'message': message.format('Active')})
+        else:
+            resp.data = json.dumps({'message': message.format('Not Present')})
+
+        resp.content_type = 'application/json'
+        resp.status = HTTP_200
+
+class Armada(object):
+    '''
+    apply armada endpoint service
+    '''
+
+    def on_post(self, req, resp):
+        armada = armadaHandler(req.stream.read())
+        print armada.tiller.k8s.get_namespace_pod()
+        armada.sync()
+
+        resp.data = json.dumps({'message': 'Success'})
+        resp.content_type = 'application/json'
+        resp.status = HTTP_200
+
+
+wsgi_app = api = falcon.API()
+
+# Routing
+
+url_routes = (
+    ('/tiller/status', Tiller()),
+    ('/apply', Armada()),
+)
+
+for route, service in url_routes:
+    api.add_route(route, service)
+
+
+setup_logging(False)