systest jenkins pipeline 12/1912/6
authorMike Marchetti <mmarchetti@sandvine.com>
Wed, 31 May 2017 20:33:02 +0000 (16:33 -0400)
committerMike Marchetti <mmarchetti@sandvine.com>
Tue, 6 Jun 2017 01:01:35 +0000 (21:01 -0400)
Change-Id: I496980ad1dcad1be691587caec41d7d86259a55f
Signed-off-by: Mike Marchetti <mmarchetti@sandvine.com>
installers/install_osm.sh
jenkins/host/start_build
jenkins/system/Jenkinsfile [new file with mode: 0644]
jenkins/system/SETTINGS [new file with mode: 0644]
jenkins/system/delete_old_containers.sh [new file with mode: 0755]
jenkins/system/start_build [new file with mode: 0755]
systest/Dockerfile [new file with mode: 0644]
systest/Jenkinsfile [new file with mode: 0644]
systest/Makefile

index f739724..34494ec 100755 (executable)
@@ -129,6 +129,32 @@ function update(){
     echo
 }
 
+function so_is_up(){
+    SO_IP=$1
+    time=0
+    step=5
+    timelength=300
+    while [ $time -le $timelength ]
+    do
+        curl -k https://$SO_IP:8008/api/operational/vcs/info \
+              --header 'accept: application/vnd.yang.data+json' \
+              --header 'authorization: Basic YWRtaW46YWRtaW4=' \
+              --header 'cache-control: no-cache' \
+              --header 'content-type: application/vnd.yang.data+json' &> /dev/null
+        RET=$?
+        if [ "$RET" == 0 ]; then
+            break
+        fi
+        sleep $step
+        echo -n "."
+        time=$((time+step))
+    done
+    if [ "$RET" != 0 ]; then
+        FATAL "OSM Failed to startup"
+    fi
+    echo
+}
+
 #Configure VCA, SO and RO with the initial configuration:
 #  RO -> tenant:osm, logs to be sent to SO
 #  VCA -> juju-password
@@ -141,7 +167,9 @@ function configure(){
     echo -e "       Configuring RO"
     lxc exec RO -- sed -i -e "s/^\#\?log_socket_host:.*/log_socket_host: $SO_CONTAINER_IP/g" /etc/osm/openmanod.cfg
     lxc exec RO -- service osm-ro restart
+
     time=0; step=2; timelength=20; while [ $time -le $timelength ]; do sleep $step; echo -n "."; time=$((time+step)); done; echo
+
     lxc exec RO -- openmano tenant-delete -f osm >/dev/null
     RO_TENANT_ID=`lxc exec RO -- openmano tenant-create osm |awk '{print $1}'`
 
@@ -153,25 +181,35 @@ function configure(){
     echo -e "       Configuring SO"
     sudo route add -host $JUJU_CONTROLLER_IP gw $VCA_CONTAINER_IP
     sudo sed -i "$ i route add -host $JUJU_CONTROLLER_IP gw $VCA_CONTAINER_IP" /etc/rc.local
-    lxc exec SO-ub -- nohup sudo -b -H /usr/rift/rift-shell -r -i /usr/rift -a /usr/rift/.artifacts -- ./demos/launchpad.py --use-xml-mode &
-    time=0; step=30; timelength=300; while [ $time -le $timelength ]; do sleep $step; echo -n "."; time=$((time+step)); done; echo
+    lxc exec SO-ub -- systemctl restart launchpad
 
-    curl -k --request POST \
+    so_is_up $SO_CONTAINER_IP
+
+    #delete existing config agent (could be there on reconfigure)
+    curl -k --request DELETE \
+      --url https://$SO_CONTAINER_IP:8008/api/config/config-agent/account/osmjuju \
+      --header 'accept: application/vnd.yang.data+json' \
+      --header 'authorization: Basic YWRtaW46YWRtaW4=' \
+      --header 'cache-control: no-cache' \
+      --header 'content-type: application/vnd.yang.data+json' &> /dev/null
+
+    result=$(curl -k --request POST \
       --url https://$SO_CONTAINER_IP:8008/api/config/config-agent \
       --header 'accept: application/vnd.yang.data+json' \
       --header 'authorization: Basic YWRtaW46YWRtaW4=' \
       --header 'cache-control: no-cache' \
       --header 'content-type: application/vnd.yang.data+json' \
-      --data '{"account": [ { "name": "osmjuju", "account-type": "juju", "juju": { "ip-address": "'$JUJU_CONTROLLER_IP'", "port": "17070", "user": "admin", "secret": "'$JUJU_PASSWD'" }  }  ]}'
+      --data '{"account": [ { "name": "osmjuju", "account-type": "juju", "juju": { "ip-address": "'$JUJU_CONTROLLER_IP'", "port": "17070", "user": "admin", "secret": "'$JUJU_PASSWD'" }  }  ]}')
+    [[ $result =~ .*success.* ]] || FATAL "Failed config-agent configuration: $result"
 
-    curl -k --request PUT \
+    result=$(curl -k --request PUT \
       --url https://$SO_CONTAINER_IP:8008/api/config/resource-orchestrator \
       --header 'accept: application/vnd.yang.data+json' \
       --header 'authorization: Basic YWRtaW46YWRtaW4=' \
       --header 'cache-control: no-cache' \
       --header 'content-type: application/vnd.yang.data+json' \
-      --data '{ "openmano": { "host": "'$RO_CONTAINER_IP'", "port": "9090", "tenant-id": "'$RO_TENANT_ID'" }, "name": "osmopenmano", "account-type": "openmano" }'
-
+      --data '{ "openmano": { "host": "'$RO_CONTAINER_IP'", "port": "9090", "tenant-id": "'$RO_TENANT_ID'" }, "name": "osmopenmano", "account-type": "openmano" }')
+    [[ $result =~ .*success.* ]] || FATAL "Failed resource-orchestrator configuration: $result"
 }
 
 function install_lxd() {
@@ -258,8 +296,11 @@ if [ -n "$SHOWOPTS" ]; then
     exit 0
 fi
 
+# if develop, we force master
 [ -z "$COMMIT_ID" ] && [ -n "$DEVELOP" ] && COMMIT_ID="master"
-[ -n "$COMMIT_ID" ] && INSTALL_FROM_SOURCE="y"
+
+# if master, force install from source
+[ -n "$COMMIT_ID" ] && [ "$COMMIT_ID" == "master" ] && INSTALL_FROM_SOURCE="y"
 
 if [ -n "$TEST_INSTALLER" ]; then
     echo -e "\nUsing local devops repo for OSM installation"
index a78d394..02bfcc2 100755 (executable)
@@ -30,6 +30,13 @@ export OSM_MDG=$1
 shift
 OSM_load_config
 
+if [ "$1" = "--build-container" ]; then
+    shift
+    [ $# -lt 1 ] && FATAL "missing container name with option --build-container"
+    export OSM_BUILD_CONTAINER=$1
+    shift
+fi
+
 if ! container_exists $OSM_BUILD_CONTAINER; then
        CONTAINER_OPTS=""
        [[ "$OSM_BUILD_CONTAINER_PRIVILEGED" == yes ]] && CONTAINER_OPTS="$CONTAINER_OPTS -c security.privileged=true"
diff --git a/jenkins/system/Jenkinsfile b/jenkins/system/Jenkinsfile
new file mode 100644 (file)
index 0000000..d3688c4
--- /dev/null
@@ -0,0 +1,30 @@
+// input parameters:
+//   boolean: BUILD_FROM_SOURCE
+//   boolean: COMMIT_ID
+//   string:  NODE
+
+node("${params.NODE}") {
+
+    stage("Setup") {
+        tag_or_branch = params.COMMIT_ID.replaceAll(/\./,"")
+        container_name_prefix = "osm-${tag_or_branch}"
+        container_name = "${container_name_prefix}-${BUILD_NUMBER}"
+    }
+
+    stage("Cleanup") {
+        // check for previous containers and clean them up
+        sh "jenkins/system/delete_old_containers.sh ${container_name_prefix}"
+    }
+
+    stage("Build") {
+        from_source = ''
+        if ( params.BUILD_FROM_SOURCE )
+        {
+            from_source = '--source'
+        }
+
+        sh "jenkins/host/start_build system --build-container ${container_name} -b ${params.COMMIT_ID} ${from_source}"
+        sh "echo ${container_name} > build_version.txt"
+        archiveArtifacts artifacts: "build_version.txt"
+    }
+}
diff --git a/jenkins/system/SETTINGS b/jenkins/system/SETTINGS
new file mode 100644 (file)
index 0000000..912799c
--- /dev/null
@@ -0,0 +1,39 @@
+#   Copyright 2016 Telefónica Investigación y Desarrollo, S.A.U.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# sample SETTINGS file
+#
+# Authors:
+#     - Gerardo Garcia
+#
+# this variable holds the name of the container image needed to build or run this product
+export OSM_BASE_IMAGE=ubuntu:16.04
+#
+# this variable must be set to allow creating the build container in privileged mode
+# this variable should be removed in the future when no privileged mode is required
+export OSM_BUILD_CONTAINER_PRIVILEGED=yes
+#
+# this variable must be set to allow ensted containers in the build container
+export OSM_BUILD_CONTAINER_ALLOW_NESTED=yes
+#
+# this variable must be set to allow creating the runtime container in privileged mode
+# this variable should be removed in the future when no privileged mode is required
+export OSM_RUNTIME_CONTAINER_PRIVILEGED=yes
+#
+# this variable must be set to allow ensted containers in the build container
+export OSM_RUNTIME_CONTAINER_ALLOW_NESTED=yes
+#
+#
+export OSM_BUILD_CONTAINER=osm
+export OSM_RUNTIME_CONTAINER=osm
diff --git a/jenkins/system/delete_old_containers.sh b/jenkins/system/delete_old_containers.sh
new file mode 100755 (executable)
index 0000000..13c6df4
--- /dev/null
@@ -0,0 +1,32 @@
+#!/bin/bash
+#   Copyright 2017 Sandvine
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# A helper routine that cleans up old container images based
+# on an input prefix to check. Jenkins builds will add an incrementing
+# build suffix  (build number) to the prefix
+#
+#$1 container prefix name
+
+keep_number=1
+prefix=$1
+
+# keep the first build
+keep=$(lxc list | grep $prefix | awk '{print $2}' | sort -rn | head -n$keep_number)
+for container in $(lxc list | grep $prefix | awk '{print $2}' | sort -rn); do
+    if [ "$container" != "$keep" ]; then
+        echo "deleting old container $container"
+        lxc delete $container --force
+    fi
+done
diff --git a/jenkins/system/start_build b/jenkins/system/start_build
new file mode 100755 (executable)
index 0000000..a50a5a8
--- /dev/null
@@ -0,0 +1,77 @@
+#!/bin/bash
+#   Copyright 2017
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# 01 May 2017 -- Michael Marchetti -- adapted from template
+#
+
+HERE=$(realpath $(dirname $0))
+OSM_JENKINS=$(dirname $HERE)
+. $OSM_JENKINS/common/all_funcs
+
+INFO "Installing packages"
+apt-get update
+
+INFO "Configuring LXD"
+# ZFS doesn't work inside a nested container. ZFS should be configured in the host LXD.
+lxd init --auto
+lxd waitready
+systemctl stop lxd-bridge
+systemctl --system daemon-reload
+cat <<EOF > /etc/default/lxd-bridge
+USE_LXD_BRIDGE="true"
+LXD_BRIDGE="lxdbr0"
+UPDATE_PROFILE="true"
+LXD_CONFILE=""
+LXD_DOMAIN="lxd"
+LXD_IPV4_ADDR="10.44.126.1"
+LXD_IPV4_NETMASK="255.255.255.0"
+LXD_IPV4_NETWORK="10.44.126.1/24"
+LXD_IPV4_DHCP_RANGE="10.44.126.2,10.44.126.254"
+LXD_IPV4_DHCP_MAX="252"
+LXD_IPV4_NAT="true"
+LXD_IPV6_ADDR=""
+LXD_IPV6_MASK=""
+LXD_IPV6_NETWORK=""
+LXD_IPV6_NAT="false"
+LXD_IPV6_PROXY="false"
+EOF
+
+systemctl enable lxd-bridge
+systemctl start lxd-bridge
+
+apt-get install -y python-pip python python-pycurl charm-tools python-pytest
+
+# TODO: use package when available on osm repo
+git clone https://osm.etsi.org/gerrit/osm/osmclient
+pip install osmclient/.
+
+devops/installers/install_osm.sh $*
+RC=$?
+
+# workaround.  for upload packages, lxdbr0 needs to be promiscuous
+# as the upload calls back to the UI server so the lxdbr0 needs
+# to operate as a bridge
+ifconfig lxdbr0 promisc
+
+if [ $RC == 0 ]; then
+   # success. find all the resulting containers
+   . devops/installers/export_ips
+
+   TO_ADD="export OSM_HOSTNAME=$SO_CONTAINER_IP"
+   grep -q OSM_HOSTNAME ~/.bashrc && sed -i "s/.*OSM_HOSTNAME.*/$TO_ADD/" ~/.bashrc || echo -e "$TO_ADD\n$(cat ~/.bashrc)" > ~/.bashrc
+fi
+
+INFO "done, RC=$RC"
+exit $RC
diff --git a/systest/Dockerfile b/systest/Dockerfile
new file mode 100644 (file)
index 0000000..ec30486
--- /dev/null
@@ -0,0 +1,6 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get -y install python \
+    libcurl4-gnutls-dev libgnutls-dev \
+    python-setuptools python-pip git python-pytest \
+    charm-tools
diff --git a/systest/Jenkinsfile b/systest/Jenkinsfile
new file mode 100644 (file)
index 0000000..8e290e8
--- /dev/null
@@ -0,0 +1,59 @@
+// input parameters:
+//   string:  UPSTREAM_PROJECT
+//   string:  NODE
+//
+//   OpenStack VIM Credentials
+//   string:  OS_AUTH_URL
+//   string:  OS_USERNAME
+//   string:  OS_PASSWORD
+//   string:  OS_PROJECT_NAME
+
+node("${params.NODE}") {
+
+    // grab the upstream artifact name
+    step ([$class: 'CopyArtifact',
+          projectName: params.UPSTREAM_PROJECT])
+
+    container_name = sh(returnStdout: true, script: 'cat build_version.txt').trim()
+
+    stage("get osm") {
+        // get the IP of the osm container
+        OSM_IP = sh(returnStdout: true, script: "lxc list ${container_name} -c 4|grep eth0 |awk '{print \$2}'").trim()
+    }
+
+    stage("checkout") {
+        checkout scm
+    }
+
+    // build the pytest container
+    stage("build-docker") {
+        sh 'docker build -t osmclient systest/.'
+    }
+
+    os_credentials = "OS_AUTH_URL=${params.OS_AUTH_URL} OS_USERNAME=${params.OS_USERNAME} OS_PASSWORD=${params.OS_PASSWORD} OS_PROJECT_NAME=${params.OS_PROJECT_NAME}"
+
+    // now run the built container.
+    withDockerContainer('osmclient') {
+
+        // install the osmclient
+        stage("install-osmclient") {
+            sh 'pip install git+https://osm.etsi.org/gerrit/osm/osmclient'
+        }
+
+        stage("build-descriptors") {
+            sh "make -C systest descriptors"
+        }
+
+        stage("smoke-test") {
+            sh "make -C systest OSM_HOSTNAME=${OSM_IP} smoke"
+            junit 'systest/reports/pytest-smoke.xml'
+        }
+
+        stage("cirros-test") {
+            sh """
+               make -C systest OSM_HOSTNAME=${OSM_IP} ${os_credentials} cirros
+               """
+            junit 'systest/reports/pytest-cirros.xml'
+        }
+    }
+}
index fb296cc..fe75754 100644 (file)
 # the make invocation.
 # eg. 
 #    export OSM_HOSTNAME=1.2.3.4:8008
-#    export OS_URL=https://<keystoneserver>:5000/v2.0
+#    export OS_AUTH_URL=https://<keystoneserver>:5000/v2.0
 #    export OS_USERNAME=admin
 #    export OS_PASSWORD=admin
 #    export OS_PROJECT_NAME=admin
 OSM_HOSTNAME ?=
-OS_URL ?=
+OS_AUTH_URL ?=
 OS_USERNAME ?=
 OS_PASSWORD_NAME ?=
 OS_PROJECT_NAME ?=
 
-ifdef OS_URL
-    OPTION_OS_URL=--os-url $(OS_URL)
+ifdef OS_AUTH_URL
+    OPTION_OS_AUTH_URL=--os-url $(OS_AUTH_URL)
 endif
 ifdef OS_USERNAME
     OPTION_OS_USERNAME=--os-username $(OS_USERNAME)
@@ -48,8 +48,9 @@ ifdef TEST_NSD_DESCRIPTORS
     OPTION_TEST_NSD_DESCRIPTORS=--osm-nsd-descriptor-packages $(TEST_NSD_DESCRIPTORS)
 endif
 
-DESCRIPTOR_REPO_NAME=descriptor-packages
-DESCRIPTOR_BUILD_DIR := $(shell pwd)/$(DESCRIPTOR_REPO_NAME)/build
+DESCRIPTOR_REPO_NAME = descriptor-packages
+DESCRIPTOR_REPO_DIR ?= $(shell pwd)/descriptor-packages
+DESCRIPTOR_BUILD_DIR := $(DESCRIPTOR_REPO_DIR)/build
 OPTION_DESCRIPTOR_BUILD_DIR=--osm-descriptor-packages $(DESCRIPTOR_BUILD_DIR)
 
 TEST_OSM_NS_NAME_PREFIX=pytest-$(shell date +%D-%T)-
@@ -79,24 +80,24 @@ check_OSM_HOSTNAME:
        $(call check_env_var,OSM_HOSTNAME)
 
 check_openstack_env:
-       $(call check_env_var,OS_URL)
+       $(call check_env_var,OS_AUTH_URL)
        $(call check_env_var,OS_USERNAME)
        $(call check_env_var,OS_PASSWORD)
        $(call check_env_var,OS_PROJECT_NAME)
 
 .PHONY: check_openstack_env check_OSM_HOSTNAME
 
-$(DESCRIPTOR_REPO_NAME):
-       @test -e $(DESCRIPTOR_REPO_NAME) || git clone $(DESCRIPTOR_REPO)
-       make -C $(DESCRIPTOR_REPO_NAME)
+descriptors:
+       test -e $(DESCRIPTOR_REPO_NAME) || git clone $(DESCRIPTOR_REPO)
+       $(MAKE) -C $(DESCRIPTOR_REPO_NAME)
 
 report_dir:
        @mkdir -p reports
 
-_run_test: $(DESCRIPTOR_REPO_NAME) report_dir
+_run_test: report_dir
        $(Q)py.test \
         --osmhost $(OSM_HOSTNAME) \
-        $(OPTION_OS_URL) \
+        $(OPTION_OS_AUTH_URL) \
         $(OPTION_OS_USERNAME) \
         $(OPTION_OS_PASSWORD) \
         $(OPTION_OS_PROJECT_NAME) \