Initial draft for airship-seaworthy pipeline

Change-Id: Ifdd3b1e18eaa03809d311d31357a6d629e03f786

tools/gate/Jenkinsfile

@@ -0,0 +1,415 @@
+
+// Pipeline expects genesis to be setup according to the documentation
+// including network, ntp, ip rules, etc.
+// http://treasuremap.readthedocs.io/en/latest/deployment.html#genesis-node
+
+import org.yaml.snakeyaml.Yaml
+import org.yaml.snakeyaml.DumperOptions;
+
+import groovy.json.JsonSlurperClassic
+import groovy.json.JsonOutput
+
+PEGLEG_IMAGE='quay.io/airshipit/pegleg:54e30687d444e83cbc628a2dfb4bb3cb0815328f'
+
+KEYSTONE_URL = 'http://keystone.ucp.svc.cluster.local:80/v3/auth/tokens'
+SHIPYARD_URL = 'http://shipyard-api.ucp.svc.cluster.local/api/v1.0'
+
+ARTF_PATH="cicd/${JOB_BASE_NAME}/${BUILD_NUMBER}"
+
+SITE_NAME='airship-seaworthy'
+SHIPYARD_CREDS = 'airship-seaworthy-shipyard-iam-pw'
+IPMI_CREDS = 'airship-seaworthy-ipmi'
+GENESIS_NODE = 'airship-seaworthy-genesis'
+
+GENESIS_IPMI_IP = '10.23.104.11'
+GENESIS_COREDNS_IP = '10.23.22.11'
+GENESIS_CEPH_DISKS = ['c', 'd', 'e', 'f', 'g', 'h']
+
+
+IPMI_IPS = ['10.23.104.12',
+            '10.23.104.13',
+            '10.23.104.14',
+            '10.23.104.17',
+            '10.23.104.19']
+
+if (env.GERRIT_REFSPEC) {
+    AIRSHIP_MANIFESTS = GERRIT_REFSPEC
+}
+
+
+//// container utils
+
+def pod_config = {
+    return """
+    apiVersion: v1
+    kind: Pod
+    spec:
+      securityContext:
+        runAsUser: 10000
+    """
+}
+
+
+//// artf utils
+
+def artf = Artifactory.server 'artifactory'
+
+def artf_spec = { pattern, target ->
+    spec = ['files': [['pattern': pattern,
+                       'target': target,
+                       'flat': true]]]
+    return new JsonOutput().toJson(spec)
+}
+
+def artf_publish = { pattern, target ->
+    info = artf.upload(artf_spec(pattern, target))
+    artf.publishBuildInfo(info)
+}
+
+def artf_download = { pattern, target ->
+    artf.download(artf_spec(pattern, target))
+}
+
+
+//// manifest utils
+
+def pegleg_collect = {
+    stage ('Pegleg Collect') {
+        sh 'mkdir -p manifests bundle'
+        sh 'pegleg site -p airship-seaworthy collect airship-seaworthy -s output'
+
+        sh 'tar czf manifests.tar.gz manifests'
+        artf_publish('manifests.tar.gz', "${ARTF_PATH}/")
+    }
+}
+
+def prom_gen = {
+    stage ("Promenade Certs Gen") {
+        // note: certs are generated for verification purpose only
+        // deployment will use certs merged into security repo
+        sh 'mkdir certs'
+        sh 'promenade generate-certs -o certs/ manifests/*.yaml'
+    }
+    stage ("Promenade Bundle Gen") {
+        sh "promenade build-all --validators -o bundle manifests/*.yaml"
+
+        sh 'tar czf bundle.tar.gz bundle'
+        artf_publish('bundle.tar.gz', "${ARTF_PATH}/")
+    }
+}
+
+
+//// prepare manifests
+
+def process_manifests = {
+    vm2() {
+        clone('openstack/airship-treasuremap', AIRSHIP_MANIFESTS)
+
+        stage ('Bare-Metal Reset') {
+
+            sh 'apt-get update'
+            sh 'apt-get install -y ipmitool'
+
+            withCredentials([usernamePassword(credentialsId: IPMI_CREDS,
+                                          usernameVariable: 'IUSER',
+                                          passwordVariable: 'IPASS')]) {
+                IPMI_IPS.each() {
+                    withCredentials([usernamePassword(credentialsId: IPMI_CREDS,
+                                                  usernameVariable: 'IUSER',
+                                                  passwordVariable: 'IPASS')]) {
+                        sh "ipmitool -I lanplus -H ${it} -U \$IUSER -P \$IPASS chassis power off"
+                    }
+                }
+
+                sh "ipmitool -I lanplus -H ${GENESIS_IPMI_IP} -U \$IUSER -P \$IPASS chassis power off"
+                sh "ipmitool -I lanplus -H ${GENESIS_IPMI_IP} -U \$IUSER -P \$IPASS chassis power on"
+            }
+        }
+
+        stage ('Resolve Versions') {
+
+            def versions = readYaml file: 'global/v4.0/software/config/versions.yaml'
+            PROMENADE_IMAGE = versions.data.images.ucp.promenade.promenade
+
+            def shipyard = readYaml file: "site/${SITE_NAME}/secrets/passphrases/ucp_shipyard_keystone_password.yaml"
+            SHIPYARD_PASSWD = shipyard.data
+
+            def ipmi = readYaml file: "site/${SITE_NAME}/secrets/passphrases/ipmi_admin_password.yaml"
+            IPMI_PASSWD = ipmi.data
+        }
+
+        pegleg_collect()
+        prom_gen()
+    }
+}
+
+
+//// genesis utils
+
+def genesis_wait = {
+    stage('Genesis Wait') {
+        timeout(12) {
+            node(GENESIS_NODE) {
+                sh 'hostname'
+            }
+        }
+    }
+}
+
+def genesis_cleanup = {
+    stage('Genesis Cleanup') {
+        sh "sudo -S airship-seaworthy/tools/cleanup.sh -f"
+
+        GENESIS_CEPH_DISKS.each() {
+            sh "sudo parted -s /dev/sd${it} mklabel msdos"
+        }
+        sh 'sudo rm -rf /var/lib/ceph/journal/osd'
+
+        // reset hosts dns servers
+        sh "sudo sed -i '/${GENESIS_COREDNS_IP}/d' /etc/resolv.conf"
+    }
+}
+
+def genesis_run = {
+    stage('Genesis Deploy') {
+        artf_download("${ARTF_PATH}/bundle.tar.gz", "")
+        sh 'tar xzf bundle.tar.gz'
+
+        dir ('bundle') {
+            timeout (90) {
+                sh 'sudo ./genesis.sh'
+                sh 'sudo -S ./validate-genesis.sh'
+            }
+        }
+    }
+
+    sh 'sudo kubectl get pods --all-namespaces -o wide'
+}
+
+def debug_report = {
+    node(GENESIS_NODE) {
+        sh "sudo debug-report.sh"
+        artf_publish("debug-cab23-r720-11.tgz", "${ARTF_PATH}/")
+    }
+}
+
+def genesis_deploy = {
+    node(GENESIS_NODE) {
+
+        genesis_cleanup()
+
+        try {
+            genesis_run()
+            sleep 47 // wait for k8s to calm down
+
+        } catch (err) {
+            print err
+            debug_report()
+            error(err)
+        }
+    }
+}
+
+
+//// keystone utils
+
+def keystone_token = {
+
+    withCredentials([[$class: 'UsernamePasswordMultiBinding',
+                      credentialsId: SHIPYARD_CREDS,
+                      usernameVariable: 'user',
+                      passwordVariable: 'pass']]) {
+
+        def req = ['auth': [
+                   'identity': [
+                     'methods': ['password'],
+                     'password': [
+                       'user': ['name': user,
+                                'domain': ['id': 'default'],
+                                'password': pass]]]]]
+
+        def jreq = new JsonOutput().toJson(req)
+
+        def res = httpRequest (url: KEYSTONE_URL,
+                               contentType: 'APPLICATION_JSON',
+                               httpMode: 'POST',
+                               requestBody: jreq)
+
+        return res.getHeaders()['X-Subject-Token'][0]
+    }
+}
+
+
+//// shipyard utils
+
+def shipyard_config_upload = {
+
+    def token = keystone_token()
+
+    stage('Shipyard Config Create') {
+
+        artf_download("${ARTF_PATH}/manifests.tar.gz", "")
+        sh "tar xzf manifests.tar.gz"
+
+        def manifests = readFile 'manifests/airship-manifests.yaml'
+
+        def res = httpRequest (url: "${SHIPYARD_URL}/configdocs/${SITE_NAME}?buffermode=replace",
+                               httpMode: 'POST',
+                               customHeaders: [[name: 'Content-Type', value: 'application/x-yaml'],
+                                               [name: 'X-Auth-Token', value: token]],
+                               requestBody: manifests)
+
+        if (res.status != 201) {
+            error("Failed to upload configdocs: ${res.status}")
+        }
+        print res.content
+    }
+
+    stage('Shipyard Config Commit') {
+        def res = httpRequest (url: "${SHIPYARD_URL}/commitconfigdocs", httpMode: 'POST',
+                               customHeaders: [[name: 'X-Auth-Token', value: token]])
+        if (res.status != 200) {
+            error("Failed to commit configdocs: ${res.status}")
+        }
+        print res.content
+    }
+}
+
+
+def shipyard_steps_get = { action ->
+
+    def res = httpRequest (url: "${SHIPYARD_URL}/actions/${action}",
+                           contentType: 'APPLICATION_JSON',
+                           httpMode: 'GET',
+                           customHeaders: [[name: 'X-Auth-Token', value: keystone_token()]])
+
+    if (res.status != 200) {
+        error("Failed to get Shypyard action steps: ${res.status}")
+    }
+
+    def cont = new JsonSlurperClassic().parseText(res.content)
+    print cont
+
+    return cont.steps
+}
+
+
+def shipyard_step_wait = { systep, interval ->
+
+    print ">> Waiting on Shipyard step: ${systep}"
+
+    def String state = systep.state
+    def errcount = 0
+    def res
+
+    while (state == null || state == 'running' || state == 'queued' || state == 'scheduled') {
+        sleep interval
+
+        if (errcount > 3) {
+            print "Multiple re-tries done already - giving up!"
+            break
+        }
+
+        res = httpRequest (url: "${SHIPYARD_URL}${systep.url}",
+                               contentType: 'APPLICATION_JSON',
+                               httpMode: 'GET',
+                               customHeaders: [[name: 'X-Auth-Token', value: keystone_token()]])
+
+        if (!res) {
+            print "httpRequest returned null - likely library issue"
+            errcount+=1
+            continue
+        }
+
+        if (res.status != 200) {
+            print "Failed to get Shipyard step info: ${res.status}"
+            errcount+=1
+            continue
+        }
+
+        if (!res.content) {
+            print "Shypyard returned null content"
+            errcount+=1
+            continue
+        }
+
+        def cont = new JsonSlurperClassic().parseText(res.content)
+        print cont
+        state = cont.state
+    }
+
+    if (state != 'success' && state != 'skipped') {
+        error("Failed Shipyard task ${systep.id}: ${res.status}, ${res.content}")
+    }
+}
+
+def shipyard_steps_wait = { action ->
+
+    def systeps = shipyard_steps_get(action)
+
+    systeps.each() {
+        // add explicit stage for steps taking long time
+        if (it.id == 'drydock_build' || it.id == 'armada_build') {
+            stage ("Shipyard (${it.id})") {
+               shipyard_step_wait(it, 240)
+            }
+        } else {
+            shipyard_step_wait(it, 4)
+        }
+    }
+}
+
+def shipyard_action_create = { action ->
+
+    def req = [ 'name': action ]
+    def jreq = new JsonOutput().toJson(req)
+
+    res = httpRequest (url: "${SHIPYARD_URL}/actions",
+                       contentType: 'APPLICATION_JSON',
+                       httpMode: 'POST',
+                       customHeaders: [[name: 'X-Auth-Token', value: keystone_token()]],
+                       requestBody: jreq)
+
+    if (res.status != 201) {
+        error("Failed to create Shypyard action: ${res.status}")
+    }
+
+    def cont = new JsonSlurperClassic().parseText(res.content)
+    print cont
+
+    shipyard_steps_wait(cont.id)
+}
+
+def shipyard_deploy = { action ->
+    node(GENESIS_NODE) {
+        try {
+            shipyard_config_upload()
+            shipyard_action_create(action)
+
+        } catch (err) {
+            print err
+            debug_report()
+            error(err)
+        }
+
+        // debug_report()
+        sh 'sudo kubectl get pods --all-namespaces -o wide'
+    }
+}
+
+
+//// execute workflow
+
+bare_metal_reset()
+resolve_versions()
+process_manifests()
+
+genesis_wait()
+genesis_deploy()
+
+shipyard_deploy('deploy_site')
+
+// verify update on unchanged docs
+// tests general shipyard workflow on site update
+// shipyard_deploy('update_site')
+

tools/gate/seed.groovy

@@ -0,0 +1,54 @@
+
+pipelineJob('airship-seaworthy') {
+
+    displayName('Airship Seaworthy - bare-metal continuous deployment pipeline')
+    description('')
+
+    parameters {
+        string {
+            defaultValue("master")
+            description("Reference to airship-treasuremap, e.g. refs/changes/12/12345/12")
+            name("AIRSHIP_MANIFESTS")
+        }
+    }
+
+
+    concurrentBuild(false)
+
+    triggers {
+        gerritTrigger {
+            serverName('fixme-gerrit')
+            gerritProjects {
+                gerritProject {
+                    compareType('PLAIN')
+                    pattern("openstack/airship-treasuremap")
+                    branches {
+                        branch {
+                            compareType("ANT")
+                            pattern("**")
+                        }
+                    }
+                    disableStrictForbiddenFileVerification(false)
+                }
+            }
+            triggerOnEvents {
+                patchsetCreated {
+                   excludeDrafts(false)
+                   excludeTrivialRebase(false)
+                   excludeNoCodeChange(false)
+                }
+                commentAddedContains {
+                   commentAddedCommentContains('recheck')
+                }
+            }
+        }
+
+        definition {
+            cps {
+                script(readFileFromWorkspace("tools/gate/Jenkinsfile"))
+                sandbox()
+            }
+        }
+    }
+}
+