From: Admin Date: Fri, 23 Jun 2017 08:36:52 +0000 (+0200) Subject: Merge branch 'branchfrom--descriptor-packages' X-Git-Tag: v2.0.2~6 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=8f83069696a74f3fc9272a5c65d5e75e923e7ff9;hp=d21e9d06f4b662d6a7166c4715875fb41cd561ff;p=osm%2Fdevops.git Merge branch 'branchfrom--descriptor-packages' --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..d9a34988 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +.cache diff --git a/README b/README new file mode 100644 index 00000000..df4b3457 --- /dev/null +++ b/README @@ -0,0 +1,24 @@ +# Copyright 2016 RIFT.IO Inc +# Copyright 2016 Telefónica Investigación y Desarrollo S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +README for https://osm.etsi.org/gerrit/osm/devops.git + +subdirectories: + jenkins -- scripts executed by jenkins on the container host and some initial scripts executed inside the container to start a build + see jenkins/README for more + installers -- scripts to be executed to install OSM from source, builds or packages + see installers/README for more + diff --git a/installers/README b/installers/README new file mode 100644 index 00000000..43662cd2 --- /dev/null +++ b/installers/README @@ -0,0 +1,17 @@ +# Copyright 2016 Telefónica Investigación y Desarrollo S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +This directory holds the scripts and tools needed to install OSM + + diff --git a/installers/export_ips b/installers/export_ips new file mode 100644 index 00000000..d651b28a --- /dev/null +++ b/installers/export_ips @@ -0,0 +1,25 @@ +# This file is meant to be SOURCED +# +# Copyright 2016 Telefónica Investigación y Desarrollo S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# 23 Sept 2016 -- Gerardo Garcia -- Genesis + +#Get IP addresses +DEFAULT_IF=`route -n |awk '$1~/^0.0.0.0/ {print $8}'` +export DEFAULT_IP=`ip -o -4 a |grep ${DEFAULT_IF}|awk '{split($4,a,"/"); print a[1]}'` +export VCA_CONTAINER_IP=`lxc list VCA -c 4|grep eth0 |awk '{print $2}'` +export SO_CONTAINER_IP=`lxc list SO-ub -c 4|grep eth0 |awk '{print $2}'` +export RO_CONTAINER_IP=`lxc list RO -c 4|grep eth0 |awk '{print $2}'` + diff --git a/installers/install_osm.sh b/installers/install_osm.sh new file mode 100755 index 00000000..f39eb410 --- /dev/null +++ b/installers/install_osm.sh @@ -0,0 +1,393 @@ +#!/bin/bash +# Copyright 2016 Telefónica Investigación y Desarrollo S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function usage(){ + echo -e "usage: $0 [OPTIONS]" + echo -e "Install OSM from binaries or source code (by default, from binaries)" + echo -e " OPTIONS" + echo -e " --uninstall: uninstall OSM: remove the containers and delete NAT rules" + echo -e " --source: install OSM from source code using the latest stable tag" + echo -e " -r : use specified repository name for osm packages" + echo -e " -R : use specified release for osm packages" + echo -e " -u : use specified repository url for osm packages" + echo -e " -k : use specified repository public key url" + echo -e " -b : install OSM from source code using a specific branch (master, v2.0, ...) or tag" + echo -e " -b master (main dev branch)" + echo -e " -b v2.0 (v2.0 branch)" + echo -e " -b tags/v1.1.0 (a specific tag)" + echo -e " ..." + echo -e " --develop: (deprecated, use '-b master') install OSM from source code using the master branch" + echo -e " --nat: install only NAT rules" +# echo -e " --update: update to the latest stable release or to the latest commit if using a specific branch" + echo -e " --showopts: print chosen options and exit (only for debugging)" + echo -e " -y: do not prompt for confirmation, assumes yes" + echo -e " -h / --help: print this help" +} + +#Uninstall OSM: remove containers +function uninstall(){ + echo -e "\nUninstalling OSM" + if [ $RC_CLONE ] || [ -n "$TEST_INSTALLER" ]; then + $OSM_DEVOPS/jenkins/host/clean_container RO + $OSM_DEVOPS/jenkins/host/clean_container VCA + $OSM_DEVOPS/jenkins/host/clean_container SO + #$OSM_DEVOPS/jenkins/host/clean_container UI + else + lxc stop RO && lxc delete RO + lxc stop VCA && lxc delete VCA + lxc stop SO-ub && lxc delete SO-ub + fi +} + +#Configure NAT rules, based on the current IP addresses of containers +function nat(){ + echo -e "\nChecking required packages: iptables-persistent" + dpkg -l iptables-persistent &>/dev/null || ! echo -e " Not installed.\nInstalling iptables-persistent requires root privileges" || \ + sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install iptables-persistent + echo -e "\nConfiguring NAT rules" + echo -e " Required root privileges" + sudo $OSM_DEVOPS/installers/nat_osm +} + +#Update RO, SO and UI: +function update(){ + echo -e "\nUpdating components" + + echo -e " Updating RO" + CONTAINER="RO" + MDG="RO" + INSTALL_FOLDER="/opt/openmano" + echo -e " Fetching the repo" + lxc exec $CONTAINER -- git -C $INSTALL_FOLDER fetch --all + BRANCH="" + BRANCH=`lxc exec $CONTAINER -- git -C $INSTALL_FOLDER status -sb | head -n1 | sed -n 's/^## \(.*\).*/\1/p'|awk '{print $1}' |sed 's/\(.*\)\.\.\..*/\1/'` + [ -z "$BRANCH" ] && FATAL "Could not find the current branch in use in the '$MDG'" + CURRENT=`lxc exec $CONTAINER -- git -C $INSTALL_FOLDER status |head -n1` + CURRENT_COMMIT_ID=`lxc exec $CONTAINER -- git -C $INSTALL_FOLDER rev-parse HEAD` + echo " FROM: $CURRENT ($CURRENT_COMMIT_ID)" + # COMMIT_ID either was previously set with -b option, or is an empty string + CHECKOUT_ID=$COMMIT_ID + [ -z "$CHECKOUT_ID" ] && [ "$BRANCH" == "HEAD" ] && CHECKOUT_ID="tags/$LATEST_STABLE_DEVOPS" + [ -z "$CHECKOUT_ID" ] && [ "$BRANCH" != "HEAD" ] && CHECKOUT_ID="$BRANCH" + if [[ $CHECKOUT_ID == "tags/"* ]]; then + REMOTE_COMMIT_ID=`lxc exec $CONTAINER -- git -C $INSTALL_FOLDER rev-list -n 1 $CHECKOUT_ID` + else + REMOTE_COMMIT_ID=`lxc exec $CONTAINER -- git -C $INSTALL_FOLDER rev-parse origin/$CHECKOUT_ID` + fi + echo " TO: $CHECKOUT_ID ($REMOTE_COMMIT_ID)" + if [ "$CURRENT_COMMIT_ID" == "$REMOTE_COMMIT_ID" ]; then + echo " Nothing to be done." + else + echo " Update required." + lxc exec $CONTAINER -- service osm-ro stop + lxc exec $CONTAINER -- git -C /opt/openmano stash + lxc exec $CONTAINER -- git -C /opt/openmano pull --rebase + lxc exec $CONTAINER -- git -C /opt/openmano checkout $CHECKOUT_ID + lxc exec $CONTAINER -- git -C /opt/openmano stash pop + lxc exec $CONTAINER -- /opt/openmano/database_utils/migrate_mano_db.sh + lxc exec $CONTAINER -- service osm-ro start + fi + echo + + echo -e " Updating SO and UI" + CONTAINER="SO-ub" + MDG="SO" + INSTALL_FOLDER="" # To be filled in + echo -e " Fetching the repo" + lxc exec $CONTAINER -- git -C $INSTALL_FOLDER fetch --all + BRANCH="" + BRANCH=`lxc exec $CONTAINER -- git -C $INSTALL_FOLDER status -sb | head -n1 | sed -n 's/^## \(.*\).*/\1/p'|awk '{print $1}' |sed 's/\(.*\)\.\.\..*/\1/'` + [ -z "$BRANCH" ] && FATAL "Could not find the current branch in use in the '$MDG'" + CURRENT=`lxc exec $CONTAINER -- git -C $INSTALL_FOLDER status |head -n1` + CURRENT_COMMIT_ID=`lxc exec $CONTAINER -- git -C $INSTALL_FOLDER rev-parse HEAD` + echo " FROM: $CURRENT ($CURRENT_COMMIT_ID)" + # COMMIT_ID either was previously set with -b option, or is an empty string + CHECKOUT_ID=$COMMIT_ID + [ -z "$CHECKOUT_ID" ] && [ "$BRANCH" == "HEAD" ] && CHECKOUT_ID="tags/$LATEST_STABLE_DEVOPS" + [ -z "$CHECKOUT_ID" ] && [ "$BRANCH" != "HEAD" ] && CHECKOUT_ID="$BRANCH" + if [[ $CHECKOUT_ID == "tags/"* ]]; then + REMOTE_COMMIT_ID=`lxc exec $CONTAINER -- git -C $INSTALL_FOLDER rev-list -n 1 $CHECKOUT_ID` + else + REMOTE_COMMIT_ID=`lxc exec $CONTAINER -- git -C $INSTALL_FOLDER rev-parse origin/$CHECKOUT_ID` + fi + echo " TO: $CHECKOUT_ID ($REMOTE_COMMIT_ID)" + if [ "$CURRENT_COMMIT_ID" == "$REMOTE_COMMIT_ID" ]; then + echo " Nothing to be done." + else + echo " Update required." + # Instructions to be added + # lxc exec SO-ub -- ... + fi + echo +} + +function so_is_up(){ + SO_IP=$1 + time=0 + step=5 + timelength=300 + while [ $time -le $timelength ] + do + curl -k https://$SO_IP:8008/api/operational/vcs/info \ + --header 'accept: application/vnd.yang.data+json' \ + --header 'authorization: Basic YWRtaW46YWRtaW4=' \ + --header 'cache-control: no-cache' \ + --header 'content-type: application/vnd.yang.data+json' &> /dev/null + RET=$? + if [ "$RET" == 0 ]; then + break + fi + sleep $step + echo -n "." + time=$((time+step)) + done + if [ "$RET" != 0 ]; then + FATAL "OSM Failed to startup" + fi + echo +} + +#Configure VCA, SO and RO with the initial configuration: +# RO -> tenant:osm, logs to be sent to SO +# VCA -> juju-password +# SO -> route to Juju Controller, add RO account, add VCA account +function configure(){ + #Configure components + echo -e "\nConfiguring components" + . $OSM_DEVOPS/installers/export_ips + + echo -e " Configuring RO" + lxc exec RO -- sed -i -e "s/^\#\?log_socket_host:.*/log_socket_host: $SO_CONTAINER_IP/g" /etc/osm/openmanod.cfg + lxc exec RO -- service osm-ro restart + + time=0; step=2; timelength=20; while [ $time -le $timelength ]; do sleep $step; echo -n "."; time=$((time+step)); done; echo + + lxc exec RO -- openmano tenant-delete -f osm >/dev/null + RO_TENANT_ID=`lxc exec RO -- openmano tenant-create osm |awk '{print $1}'` + lxc exec RO -- sed -i '/export OPENMANO_TENANT=osm/d' .bashrc + lxc exec RO -- sed -i '$ i export OPENMANO_TENANT=osm' .bashrc + #lxc exec RO -- sh -c 'echo "export OPENMANO_TENANT=osm" >> .bashrc' + + echo -e " Configuring VCA" + JUJU_PASSWD=`date +%s | sha256sum | base64 | head -c 32` + echo -e "$JUJU_PASSWD\n$JUJU_PASSWD" | lxc exec VCA -- juju change-user-password + JUJU_CONTROLLER_IP=`lxc exec VCA -- lxc list -c 4 |grep eth0 |awk '{print $2}'` + + echo -e " Configuring SO" + sudo route add -host $JUJU_CONTROLLER_IP gw $VCA_CONTAINER_IP + sudo sed -i "$ i route add -host $JUJU_CONTROLLER_IP gw $VCA_CONTAINER_IP" /etc/rc.local + lxc exec SO-ub -- systemctl restart launchpad + + so_is_up $SO_CONTAINER_IP + + #delete existing config agent (could be there on reconfigure) + curl -k --request DELETE \ + --url https://$SO_CONTAINER_IP:8008/api/config/config-agent/account/osmjuju \ + --header 'accept: application/vnd.yang.data+json' \ + --header 'authorization: Basic YWRtaW46YWRtaW4=' \ + --header 'cache-control: no-cache' \ + --header 'content-type: application/vnd.yang.data+json' &> /dev/null + + result=$(curl -k --request POST \ + --url https://$SO_CONTAINER_IP:8008/api/config/config-agent \ + --header 'accept: application/vnd.yang.data+json' \ + --header 'authorization: Basic YWRtaW46YWRtaW4=' \ + --header 'cache-control: no-cache' \ + --header 'content-type: application/vnd.yang.data+json' \ + --data '{"account": [ { "name": "osmjuju", "account-type": "juju", "juju": { "ip-address": "'$JUJU_CONTROLLER_IP'", "port": "17070", "user": "admin", "secret": "'$JUJU_PASSWD'" } } ]}') + [[ $result =~ .*success.* ]] || FATAL "Failed config-agent configuration: $result" + + result=$(curl -k --request PUT \ + --url https://$SO_CONTAINER_IP:8008/api/config/resource-orchestrator \ + --header 'accept: application/vnd.yang.data+json' \ + --header 'authorization: Basic YWRtaW46YWRtaW4=' \ + --header 'cache-control: no-cache' \ + --header 'content-type: application/vnd.yang.data+json' \ + --data '{ "openmano": { "host": "'$RO_CONTAINER_IP'", "port": "9090", "tenant-id": "'$RO_TENANT_ID'" }, "name": "osmopenmano", "account-type": "openmano" }') + [[ $result =~ .*success.* ]] || FATAL "Failed resource-orchestrator configuration: $result" +} + +function install_lxd() { + lxd init --auto + lxd waitready + systemctl stop lxd-bridge + systemctl --system daemon-reload + systemctl enable lxd-bridge + systemctl start lxd-bridge +} + +function ask_user(){ + # ask to the user and parse a response among 'y', 'yes', 'n' or 'no'. Case insensitive + # Params: $1 text to ask; $2 Action by default, can be 'y' for yes, 'n' for no, other or empty for not allowed + # Return: true(0) if user type 'yes'; false (1) if user type 'no' + read -e -p "$1" USER_CONFIRMATION + while true ; do + [ -z "$USER_CONFIRMATION" ] && [ "$2" == 'y' ] && return 0 + [ -z "$USER_CONFIRMATION" ] && [ "$2" == 'n' ] && return 1 + [ "${USER_CONFIRMATION,,}" == "yes" ] || [ "${USER_CONFIRMATION,,}" == "y" ] && return 0 + [ "${USER_CONFIRMATION,,}" == "no" ] || [ "${USER_CONFIRMATION,,}" == "n" ] && return 1 + read -e -p "Please type 'yes' or 'no': " USER_CONFIRMATION + done +} + +UNINSTALL="" +DEVELOP="" +NAT="" +UPDATE="" +RECONFIGURE="" +TEST_INSTALLER="" +LXD="" +SHOWOPTS="" +COMMIT_ID="" +ASSUME_YES="" +INSTALL_FROM_SOURCE="" + +while getopts ":hy-:b:r:k:u:R:" o; do + case "${o}" in + h) + usage && exit 0 + ;; + b) + COMMIT_ID=${OPTARG} + ;; + r) + REPOSITORY="-r ${OPTARG}" + ;; + R) + RELEASE="-R ${OPTARG}" + ;; + k) + REPOSITORY_KEY="-k ${OPTARG}" + ;; + u) + REPOSITORY_BASE="-u ${OPTARG}" + ;; + -) + [ "${OPTARG}" == "help" ] && usage && exit 0 + [ "${OPTARG}" == "source" ] && INSTALL_FROM_SOURCE="y" && continue + [ "${OPTARG}" == "develop" ] && DEVELOP="y" && continue + [ "${OPTARG}" == "uninstall" ] && UNINSTALL="y" && continue + [ "${OPTARG}" == "nat" ] && NAT="y" && continue + [ "${OPTARG}" == "update" ] && UPDATE="y" && continue + [ "${OPTARG}" == "reconfigure" ] && RECONFIGURE="y" && continue + [ "${OPTARG}" == "test" ] && TEST_INSTALLER="y" && continue + [ "${OPTARG}" == "lxd" ] && LXD="y" && continue + [ "${OPTARG}" == "showopts" ] && SHOWOPTS="y" && continue + echo -e "Invalid option: '--$OPTARG'\n" >&2 + usage && exit 1 + ;; + \?) + echo -e "Invalid option: '-$OPTARG'\n" >&2 + usage && exit 1 + ;; + y) + ASSUME_YES="y" + ;; + *) + usage && exit 1 + ;; + esac +done + +if [ -n "$SHOWOPTS" ]; then + echo "DEVELOP=$DEVELOP" + echo "INSTALL_FROM_SOURCE=$INSTALL_FROM_SOURCE" + echo "UNINSTALL=$UNINSTALL" + echo "NAT=$NAT" + echo "UPDATE=$UPDATE" + echo "RECONFIGURE=$RECONFIGURE" + echo "TEST_INSTALLER=$TEST_INSTALLER" + echo "LXD=$LXD" + echo "SHOWOPTS=$SHOWOPTS" + echo "Install from specific refspec (-b): $COMMIT_ID" + exit 0 +fi + +# if develop, we force master +[ -z "$COMMIT_ID" ] && [ -n "$DEVELOP" ] && COMMIT_ID="master" + +# if master, force install from source +[ -n "$COMMIT_ID" ] && [ "$COMMIT_ID" == "master" ] && INSTALL_FROM_SOURCE="y" + +if [ -n "$TEST_INSTALLER" ]; then + echo -e "\nUsing local devops repo for OSM installation" + TEMPDIR="$(dirname $(realpath $(dirname $0)))" +else + echo -e "\nCreating temporary dir for OSM installation" + TEMPDIR="$(mktemp -d -q --tmpdir "installosm.XXXXXX")" + trap 'rm -rf "$TEMPDIR"' EXIT +fi + +echo -e "Checking required packages: git" +dpkg -l git &>/dev/null || ! echo -e " git not installed.\nInstalling git requires root privileges" || sudo apt-get install -y git +if [ -z "$TEST_INSTALLER" ]; then + echo -e "\nCloning devops repo temporarily" + git clone https://osm.etsi.org/gerrit/osm/devops.git $TEMPDIR + RC_CLONE=$? +fi + +echo -e "\nGuessing the current stable release" +LATEST_STABLE_DEVOPS=`git -C $TEMPDIR tag -l v[0-9].* | tail -n1` +[ -z "$COMMIT_ID" ] && [ -z "$LATEST_STABLE_DEVOPS" ] && echo "Could not find the current latest stable release" && exit 0 +echo "Latest tag in devops repo: $LATEST_STABLE_DEVOPS" +[ -z "$COMMIT_ID" ] && [ -n "$LATEST_STABLE_DEVOPS" ] && COMMIT_ID="tags/$LATEST_STABLE_DEVOPS" +[ -z "$TEST_INSTALLER" ] && git -C $TEMPDIR checkout tags/$LATEST_STABLE_DEVOPS + +OSM_DEVOPS=$TEMPDIR +OSM_JENKINS="$TEMPDIR/jenkins" +. $OSM_JENKINS/common/all_funcs + +[ -n "$UNINSTALL" ] && uninstall && echo -e "\nDONE" && exit 0 +[ -n "$NAT" ] && nat && echo -e "\nDONE" && exit 0 +[ -n "$UPDATE" ] && update && echo -e "\nDONE" && exit 0 +[ -n "$RECONFIGURE" ] && configure && echo -e "\nDONE" && exit 0 + +#Installation starts here +echo -e "\nInstalling OSM from refspec: $COMMIT_ID" +if [ -n "$INSTALL_FROM_SOURCE" ] && [ -z "$ASSUME_YES" ]; then + ! ask_user "The installation will take about 75-90 minutes. Continue (Y/n)? " y && echo "Cancelled!" && exit 1 +fi + +echo -e "\nChecking required packages: wget, curl, tar" +dpkg -l wget curl tar &>/dev/null || ! echo -e " One or several packages are not installed.\nInstalling required packages\n Root privileges are required" || sudo apt-get install -y wget curl tar + +echo -e "Checking required packages: lxd" +lxd --version &>/dev/null || FATAL "lxd not present, exiting." +[ -n "$LXD" ] && echo -e "\nConfiguring lxd" && install_lxd + +wget -q -O- https://osm-download.etsi.org/ftp/osm-2.0-two/README.txt &> /dev/null + +if [ -z "$INSTALL_FROM_SOURCE" ]; then + echo -e "\nCreating the containers and installing from binaries ..." + $OSM_DEVOPS/jenkins/host/install RO $REPOSITORY $RELEASE $REPOSITORY_KEY $REPOSITORY_BASE || FATAL "RO install failed" + $OSM_DEVOPS/jenkins/host/start_build VCA || FATAL "VCA install failed" + $OSM_DEVOPS/jenkins/host/install SO $REPOSITORY $RELEASE $REPOSITORY_KEY $REPOSITORY_BASE || FATAL "SO install failed" + $OSM_DEVOPS/jenkins/host/install UI $REPOSITORY $RELEASE $REPOSITORY_KEY $REPOSITORY_BASE || FATAL "UI install failed" +else #install from source + echo -e "\nCreating the containers and building from source ..." + $OSM_DEVOPS/jenkins/host/start_build RO --notest checkout $COMMIT_ID || FATAL "RO container build failed (refspec: '$COMMIT_ID')" + $OSM_DEVOPS/jenkins/host/start_build VCA || FATAL "VCA container build failed" + $OSM_DEVOPS/jenkins/host/start_build SO checkout $COMMIT_ID || FATAL "SO container build failed (refspec: '$COMMIT_ID')" + $OSM_DEVOPS/jenkins/host/start_build UI checkout $COMMIT_ID || FATAL "UI container build failed (refspec: '$COMMIT_ID')" +fi + +#Install iptables-persistent and configure NAT rules +nat + +#Configure components +configure + +wget -q -O- https://osm-download.etsi.org/ftp/osm-2.0-two/README2.txt &> /dev/null +echo -e "\nDONE" diff --git a/installers/nat_osm b/installers/nat_osm new file mode 100755 index 00000000..0ff4f615 --- /dev/null +++ b/installers/nat_osm @@ -0,0 +1,180 @@ +#!/bin/bash +# Copyright 2016 Telefónica Investigación y Desarrollo S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############ +# Functions +############ +usage(){ + echo -e "usage: $0 [OPTIONS]" + echo -e "Install NAT rules for OSM" + echo -e " OPTIONS" + echo -e " -u: UI/SO (rift) IP address" + echo -e " -r: RO (openmano) IP address" + echo -e " -v: VCA (juju) IP address" + echo -e " -h: show this help" +} + +################### +# End of functions +################### + +#Check root privileges +[ "$USER" != "root" ] && echo "Needed root privileges (run with sudo)" >&2 && exit 1 + +HERE=$(realpath $(dirname $0)) +OSM_DEVOPS=$(dirname $HERE) +OSM_JENKINS="$OSM_DEVOPS/jenkins" +. $OSM_JENKINS/common/all_funcs + +#Get default IP address +. $OSM_DEVOPS/installers/export_ips + +UI_IP=$DEFAULT_IP +RO_IP=$DEFAULT_IP +VCA_IP=$DEFAULT_IP + +#read input options +while getopts ":u:r:v:h-:" o; do + case "${o}" in + u) + export UI_IP="$OPTARG" + ;; + r) + export RO_IP="$OPTARG" + ;; + v) + export VCA_IP="$OPTARG" + ;; + h) + usage && exit 0 + ;; + -) + [ "${OPTARG}" == "help" ] && usage && exit 0 + echo -e "Invalid option: '--$OPTARG'\nTry $0 --help for more information" >&2 + exit 1 + ;; + \?) + echo -e "Invalid option: '-$OPTARG'\nTry $0 --help for more information" >&2 + exit 1 + ;; + :) + echo -e "Option '-$OPTARG' requires an argument\nTry $0 --help for more information" >&2 + exit 1 + ;; + *) + usage >&2 + exit -1 + ;; + esac +done + +############# +# NAT port forwarding configuration +############# +echo +echo "*** Configuring iptables rules ***" + +awk -v RO_IP="$RO_IP" -v VCA_IP="$VCA_IP" -v UI_IP="$UI_IP" -v openmano_ip="$RO_CONTAINER_IP" -v rift_ip="$SO_CONTAINER_IP" -v juju_ip="$VCA_CONTAINER_IP" ' +BEGIN {innat=0; innatpre=0; osmpre=0; donepre=0; innatpost=0; osmpost=0; donepost=0} +/^\*nat/ { + innat=1; + print; + next +} +innat==1 && /\:PREROUTING/ { + innatpre=1; + print; + next; +} +innatpre==1 && /\#Autogenerated by nat_osm/ { + osmpre=1; + next; +} +osmpre==1 && /#End autogeneration by nat_osm/ { + print "#Autogenerated by nat_osm" + print "-A PREROUTING -d "RO_IP" -p tcp -m tcp --dport 9090 -j DNAT --to-destination "openmano_ip + print "-A PREROUTING -d "UI_IP" -p tcp -m tcp --dport 8000 -j DNAT --to-destination "rift_ip + print "-A PREROUTING -d "UI_IP" -p tcp -m tcp --dport 4567 -j DNAT --to-destination "rift_ip + print "-A PREROUTING -d "UI_IP" -p tcp -m tcp --dport 8443 -j DNAT --to-destination "rift_ip + print "-A PREROUTING -d "UI_IP" -p tcp -m tcp --dport 8008 -j DNAT --to-destination "rift_ip + print "-A PREROUTING -d "UI_IP" -p tcp -m tcp --dport 80 -j DNAT --to-destination "rift_ip + #print "-A PREROUTING -d "VCA_IP" -p tcp -m tcp --dport 443 -j DNAT --to-destination "juju_ip + #print "-A PREROUTING -d "VCA_IP" -p tcp -m tcp --dport 17070 -j DNAT --to-destination "juju_ip + print "#End autogeneration by nat_osm" + osmpre=0; + donepre=1; + next; +} +osmpre==1 {next;} +innatpre==1 && /\:INPUT/ { + innatpre=0; + if (donepre==0) { + print "#Autogenerated by nat_osm" + print "-A PREROUTING -d "RO_IP" -p tcp -m tcp --dport 9090 -j DNAT --to-destination "openmano_ip + print "-A PREROUTING -d "UI_IP" -p tcp -m tcp --dport 8000 -j DNAT --to-destination "rift_ip + print "-A PREROUTING -d "UI_IP" -p tcp -m tcp --dport 4567 -j DNAT --to-destination "rift_ip + print "-A PREROUTING -d "UI_IP" -p tcp -m tcp --dport 8443 -j DNAT --to-destination "rift_ip + print "-A PREROUTING -d "UI_IP" -p tcp -m tcp --dport 8008 -j DNAT --to-destination "rift_ip + print "-A PREROUTING -d "UI_IP" -p tcp -m tcp --dport 80 -j DNAT --to-destination "rift_ip + #print "-A PREROUTING -d "VCA_IP" -p tcp -m tcp --dport 443 -j DNAT --to-destination "juju_ip + #print "-A PREROUTING -d "VCA_IP" -p tcp -m tcp --dport 17070 -j DNAT --to-destination "juju_ip + print "#End autogeneration by nat_osm" + donepre=1; + } + print; + next; +} + +innat==1 && /\:POSTROUTING/ { + innatpost=1; + print; + next; +} +innatpost==1 && /\#Autogenerated by nat_osm/ { + osmpost=1; + next; +} +osmpost==1 && /#End autogeneration by nat_osm/ { + print "#Autogenerated by nat_osm" + print "-A POSTROUTING -s "rift_ip"/24 -d "rift_ip" -p tcp --dport 8443 -j MASQUERADE" + #print "-A POSTROUTING -s "rift_ip" -p tcp -m tcp --dport 9090 -d "openmano_ip" -j SNAT --to "UI_IP + #print "-A POSTROUTING -s "rift_ip" -p tcp -m tcp --dport 17070 -d "juju_ip" -j SNAT --to "UI_IP + print "#End autogeneration by nat_osm" + osmpost=0; + donepost=1; + next; +} +osmpost==1 {next;} +innatpost==1 && /COMMIT/ { + innatpost=0; + innat=0; + if (donepost==0) { + print "#Autogenerated by nat_osm" + print "-A POSTROUTING -s "rift_ip"/24 -d "rift_ip" -p tcp --dport 8443 -j MASQUERADE" + #print "-A POSTROUTING -s "rift_ip" -p tcp -m tcp --dport 9090 -d "openmano_ip" -j SNAT --to "UI_IP + #print "-A POSTROUTING -s "rift_ip" -p tcp -m tcp --dport 17070 -d "juju_ip" -j SNAT --to "UI_IP + print "#End autogeneration by nat_osm" + donepost=1; + } + print; + next; +} +{ + print +} +' /etc/iptables/rules.v4 > testfile.tmp && mv testfile.tmp /etc/iptables/rules.v4 + +service netfilter-persistent restart + diff --git a/jenkins/README b/jenkins/README new file mode 100644 index 00000000..14e43b75 --- /dev/null +++ b/jenkins/README @@ -0,0 +1,24 @@ +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +This directory holds the scripts and tools needed by jenkins to run jobs + +The 'host' subdirectory is meant to be invoked directly by jenkins either in its own container or on the host. + +The 'template' subdirectory contains sample files needed to configure a new MDG + +The other subdirectories correspond to MDGs and contain the scripts that are run inside the container + +documentation is in the OSM wiki. See https://osm.etsi.org/wiki/index.php/Jenkins_Build_Scripts + diff --git a/jenkins/RO/SETTINGS b/jenkins/RO/SETTINGS new file mode 100644 index 00000000..8c37f162 --- /dev/null +++ b/jenkins/RO/SETTINGS @@ -0,0 +1,28 @@ +# +# Copyright 2016 Telefónica Investigación y Desarrollo, S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Gerardo Garcia - gerardo.garciadeblas@telefonica.com +# +# this variable holds the name of the container image needed to build or run this product +export OSM_BASE_IMAGE=ubuntu:16.04 +# +# this variable holds the name of the container to be used to build a package +# if this container already exists, the build process can skip the container build +export OSM_BUILD_CONTAINER=RO +# +# this variable holds the name of the container to be used to run a package +# if this container already exists, the run processes can skip the container build +export OSM_RUNTIME_CONTAINER=RO diff --git a/jenkins/RO/install b/jenkins/RO/install new file mode 100755 index 00000000..d2eaf58f --- /dev/null +++ b/jenkins/RO/install @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Copyright 2016 Telefónica Investigación y Desarrollo, S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Gerardo Garcia - gerardo.garciadeblas@telefonica.com +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +# SET YOUR MDG repository name here +export OSM_MDG=RO +OSM_load_config + +. $OSM_JENKINS/common/install_common + +# Installation starts here + +#Release TWO + +apt-get update +apt-get -y install python-lib-osm-openvim +apt-get -y install python-osm-ro + +OSMLIBOVIM_PATH=`python -c 'import lib_osm_openvim; print lib_osm_openvim.__path__[0]'` || FATAL "lib-osm-openvim was not properly installed" +OSMRO_PATH=`python -c 'import osm_ro; print osm_ro.__path__[0]'` || FATAL "osm-ro was not properly installed" +sed -i "s/\${DIRNAME}\/\.\.\/openmanod\ -v/openmanod\ -v/g" /usr/lib/python2.7/dist-packages/osm_ro/database_utils/migrate_mano_db.sh +systemctl disable osm-ro.service +sed -i "/User=/d" /etc/systemd/system/osm-ro.service +sed -i "s/ExecStart=openmanod/ExecStart=\/usr\/bin\/openmanod/" /etc/systemd/system/osm-ro.service +systemctl enable osm-ro.service + +DEBIAN_FRONTEND=noninteractive apt-get -y install mysql-server + +${OSMRO_PATH}/database_utils/install-db-server.sh --updatedb --no-install-packages || FATAL "osm-ro db installation failed" +${OSMLIBOVIM_PATH}/database_utils/install-db-server.sh -u mano -p manopw -d mano_vim_db --updatedb || FATAL "lib-osm-openvim db installation failed" +service osm-ro restart + +RC=$? +INFO "done, RC=$RC" +exit $RC + + diff --git a/jenkins/RO/start_build b/jenkins/RO/start_build new file mode 100755 index 00000000..01474eea --- /dev/null +++ b/jenkins/RO/start_build @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Copyright 2016 Telefónica Investigación y Desarrollo, S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Gerardo Garcia - gerardo.garciadeblas@telefonica.com +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +# SET YOU MDG repository name here +export OSM_MDG=RO +OSM_load_config + +NOTEST="" +if [ "$1" = "--notest" ]; then + shift + NOTEST="y" +fi + +OSM_git_checkout "$@" + +INFO "installing RO packages and dependencies from current repo (--noclone)" +./scripts/install-openmano.sh --noclone --force -q +RC=$? + +if [ -n "$NOTEST" ]; then + INFO "done, RC=$RC" + exit $RC +fi + +INFO "starting build" + +INFO " cleaning .pyc" +rm -f *.pyc + +INFO " compiling *.py" +TEMPFILE="$(mktemp -q -p . "openmanolinker.XXXXXX.py")" +trap 'rm -f "$TEMPFILE"' EXIT +for i in `ls vimconn_*.py |sed "s/\.py//"`; do echo "import $i" >> $TEMPFILE; done + +python $TEMPFILE && +python -m py_compile *.py # && + +#INFO " basic_test" && +#./test/basictest.sh --force --insert-bashrc --install-openvim --init-openvim #&& #uncomment to add new tests +# OTHER TESTS HERE + +RC=$? +INFO "done, RC=$RC" +exit $RC + + diff --git a/jenkins/SETTINGS b/jenkins/SETTINGS new file mode 100644 index 00000000..c0a2c1ff --- /dev/null +++ b/jenkins/SETTINGS @@ -0,0 +1,30 @@ +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Settings that are global to this installation +# MDG specific settings can be found in each MDG directory +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis +# +# base url for all repositories +OSM_GIT_URL=https://osm.etsi.org/gerrit/osm +# +# OSM_USE_LOCAL_DEVOPS +# +# this option disables the clone inside the +# the container and instead copies this directory tree +# into the container. Very useful for testing +# default is false +# +# OSM_USE_LOCAL_DEVOPS=true diff --git a/jenkins/SO/SETTINGS b/jenkins/SO/SETTINGS new file mode 100644 index 00000000..995e01d9 --- /dev/null +++ b/jenkins/SO/SETTINGS @@ -0,0 +1,37 @@ +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# sample SETTINGS file +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis +# +# this variable holds the name of the container image needed to build or run this product +export OSM_BASE_IMAGE=ubuntu:16.04 +# +# this variable holds the name of the container to be used to build a package +# if this container already exists, the build process can skip the container build +export OSM_BUILD_CONTAINER=SO-ub +# +# this variable must be set to allow creating the build container in privileged mode +# this variable should be removed in the future when no privileged mode is required +#export OSM_BUILD_CONTAINER_PRIVILEGED=yes +# +# this variable holds the name of the container to be used to run a package +# if this container already exists, the run processes can skip the container build +export OSM_RUNTIME_CONTAINER=SO-ub +# +# this variable must be set to allow creating the runtime container in privileged mode +# this variable should be removed in the future when no privileged mode is required +#export OSM_RUNTIME_CONTAINER_PRIVILEGED=yes + diff --git a/jenkins/SO/install b/jenkins/SO/install new file mode 100755 index 00000000..d881a42d --- /dev/null +++ b/jenkins/SO/install @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# +# Copyright 2017 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Author(s): Jeremy Mordkoff +# Creation Date: 26 April 2017 +# +# + +# INSTALL.sh +# install launchpad SO and all of its dependencies +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/install_common + +# Defensive bash programming flags +set -o errexit # Exit on any error +trap 'echo ERROR: Command failed: \"$BASH_COMMAND\"' ERR +set -o nounset # Expanding an unset variable is an error. Variables must be + # set before they can be used. + +############################################################################### +# Set up repo and version +PLATFORM_REPOSITORY=OSM +PLATFORM_VERSION=4.4.2.1.61839 + +while getopts ":P:V:h" o; do + case "${o}" in + P) + PLATFORM_REPOSITORY=${OPTARG} + ;; + V) + PLATFORM_VERSION=${OPTARG} + ;; + h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + +function usage() { + echo + echo "NAME:" + echo " $0" + echo + echo "SYNOPSIS:" + echo " $0 -h|--help" + echo " $0 -P -V " + echo + echo "DESCRIPTION:" + echo " Prepare current system to run SO and UI." + echo + echo " PLATFORM_REPOSITORY (optional): name of the RIFT.ware repository." + echo " PLATFORM_VERSION (optional): version of the platform packages to be installed." + echo +} + + +############################################################################### +# Main block + +# enable the right repos +curl http://repos.riftio.com/public/xenial-riftware-public-key | apt-key add - +curl -o /etc/apt/sources.list.d/RIFT.list http://buildtracker.riftio.com/repo_file/ub16/${PLATFORM_REPOSITORY}/ + +# Make the Rift repo a higher priority to work around version conflicts. +cat < /etc/apt/preferences.d/rift +Package: * +Pin: origin repos.riftio.com +Pin-Priority: 600 +EOF + +apt-get update + +# and install the tools +apt remove -y rw.toolchain-rwbase tcpdump +apt-get install -y --allow-downgrades rw.tools-container-tools=${PLATFORM_VERSION} rw.tools-scripts=${PLATFORM_VERSION} python +/usr/rift/container_tools/mkcontainer --modes ext --modes platform --rw-version ${PLATFORM_VERSION} +pip3 install lxml==3.4.0 + +chmod 777 /usr/rift /usr/rift/usr/share + +# now disable the RIFT OSM repo so that we'll get MANO from +# the OSM repos +rm -f /etc/apt/sources.list.d/RIFT.list + +apt-get update + +apt-get install -y \ + rw.core.mano-rwcal_yang_ylib-1.0 \ + rw.core.mano-rwconfig_agent_yang_ylib-1.0 \ + rw.core.mano-rwlaunchpad_yang_ylib-1.0 \ + rw.core.mano-mano_yang_ylib-1.0 \ + rw.core.mano-common-1.0 \ + rw.core.mano-rwsdn_yang_ylib-1.0 \ + rw.core.mano-rwsdnal_yang_ylib-1.0 \ + rw.core.mano-mano-types_yang_ylib-1.0 \ + rw.core.mano-rwcal-cloudsim-1.0 \ + rw.core.mano-rwcal-1.0 \ + rw.core.mano-rw_conman_yang_ylib-1.0 \ + rw.core.mano-rwcalproxytasklet-1.0 \ + rw.core.mano-rwlaunchpad-1.0 \ + rw.core.mano-rwcal-openmano-vimconnector-1.0 \ + rw.core.mano-lpmocklet_yang_ylib-1.0 \ + rw.core.mano-rwmon-1.0 \ + rw.core.mano-rwcloud_yang_ylib-1.0 \ + rw.core.mano-rwcal-openstack-1.0 \ + rw.core.mano-rw.core.mano_foss \ + rw.core.mano-rwmon_yang_ylib-1.0 \ + rw.core.mano-rwcm-1.0 \ + rw.core.mano-rwcal-mock-1.0 \ + rw.core.mano-rwcal-cloudsimproxy-1.0 \ + rw.core.mano-models-1.0 \ + rw.core.mano-rwcal-aws-1.0 diff --git a/jenkins/SO/start_build b/jenkins/SO/start_build new file mode 100755 index 00000000..a672e149 --- /dev/null +++ b/jenkins/SO/start_build @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# TEMPLATE script to start a build. This is run inside a container +# +# 6 July 2016 -- Jeremy.Mordkoff@riftio.com -- adapted from the riftware version +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +# SET YOU MDG repository name here +export OSM_MDG=SO +OSM_load_config +OSM_git_checkout "$@" + +trap 'WARNING "INTERRUPT"; exit 1' INT + +INFO "starting build" +make clean || FATAL "make clean failed" +./BUILD.sh + +RC=$? + +INFO "done, RC=$RC" +exit $RC + + diff --git a/jenkins/UI/SETTINGS b/jenkins/UI/SETTINGS new file mode 100644 index 00000000..995e01d9 --- /dev/null +++ b/jenkins/UI/SETTINGS @@ -0,0 +1,37 @@ +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# sample SETTINGS file +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis +# +# this variable holds the name of the container image needed to build or run this product +export OSM_BASE_IMAGE=ubuntu:16.04 +# +# this variable holds the name of the container to be used to build a package +# if this container already exists, the build process can skip the container build +export OSM_BUILD_CONTAINER=SO-ub +# +# this variable must be set to allow creating the build container in privileged mode +# this variable should be removed in the future when no privileged mode is required +#export OSM_BUILD_CONTAINER_PRIVILEGED=yes +# +# this variable holds the name of the container to be used to run a package +# if this container already exists, the run processes can skip the container build +export OSM_RUNTIME_CONTAINER=SO-ub +# +# this variable must be set to allow creating the runtime container in privileged mode +# this variable should be removed in the future when no privileged mode is required +#export OSM_RUNTIME_CONTAINER_PRIVILEGED=yes + diff --git a/jenkins/UI/install b/jenkins/UI/install new file mode 100755 index 00000000..4ba71233 --- /dev/null +++ b/jenkins/UI/install @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Copyright 2017 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Author(s): Jeremy Mordkoff +# Creation Date: 26 April 2017 +# +# + +# INSTALL.sh +# install launchpad UI and create the service +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/install_common + +rm -f /etc/apt/sources.list.d/RIFT.list +apt-get update + +apt-get install -y rw.ui-skyquake + +echo "Creating Service ...." +/usr/rift/bin/create_launchpad_service + diff --git a/jenkins/UI/start_build b/jenkins/UI/start_build new file mode 100755 index 00000000..50b2e06d --- /dev/null +++ b/jenkins/UI/start_build @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# TEMPLATE script to start a build. This is run inside a container +# +# 6 July 2016 -- Jeremy.Mordkoff@riftio.com -- adapted from the riftware version +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +# SET YOU MDG repository name here +export OSM_MDG=UI +OSM_load_config +OSM_git_checkout "$@" + + +INFO "starting build" +make clean || FATAL "Make clean failed" +make -j16 || FATAL "Make failed" +sudo make install || FATAL "Make install Failed" + +INFO "build done" +exit 0 + + diff --git a/jenkins/VCA/SETTINGS b/jenkins/VCA/SETTINGS new file mode 100644 index 00000000..ded50438 --- /dev/null +++ b/jenkins/VCA/SETTINGS @@ -0,0 +1,44 @@ +# Copyright 2016 Telefónica Investigación y Desarrollo, S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# sample SETTINGS file +# +# Authors: +# - Gerardo Garcia +# +# this variable holds the name of the container image needed to build or run this product +export OSM_BASE_IMAGE=ubuntu:16.04 +# +# this variable holds the name of the container to be used to build a package +# if this container already exists, the build process can skip the container build +export OSM_BUILD_CONTAINER=VCA +# +# this variable must be set to allow creating the build container in privileged mode +# this variable should be removed in the future when no privileged mode is required +export OSM_BUILD_CONTAINER_PRIVILEGED=yes +# +# this variable must be set to allow ensted containers in the build container +export OSM_BUILD_CONTAINER_ALLOW_NESTED=yes +# +# this variable holds the name of the container to be used to run a package +# if this container already exists, the run processes can skip the container build +export OSM_RUNTIME_CONTAINER=VCA +# +# this variable must be set to allow creating the runtime container in privileged mode +# this variable should be removed in the future when no privileged mode is required +export OSM_RUNTIME_CONTAINER_PRIVILEGED=yes +# +# this variable must be set to allow ensted containers in the build container +export OSM_RUNTIME_CONTAINER_ALLOW_NESTED=yes +# diff --git a/jenkins/VCA/start_build b/jenkins/VCA/start_build new file mode 100755 index 00000000..d6ccfd8c --- /dev/null +++ b/jenkins/VCA/start_build @@ -0,0 +1,87 @@ +#!/bin/bash +# Copyright 2016 Telefónica Investigación y Desarrollo S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# 20 Sep 2016 -- Gerardo Garcia -- adapted from template +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +INFO "Installing packages" +add-apt-repository -y ppa:juju/stable +apt-get update +apt-get install -y juju lxd squid-deb-proxy + +echo 'streams.canonical' > /etc/squid-deb-proxy/mirror-dstdomain.acl.d/20-juju-streams +service squid-deb-proxy reload + +INFO "Configuring LXD" +# ZFS doesn't work inside a nested container. ZFS should be configured in the host LXD. +lxd init --auto +lxd waitready +systemctl stop lxd-bridge +systemctl --system daemon-reload +cat < /etc/default/lxd-bridge +USE_LXD_BRIDGE="true" +LXD_BRIDGE="lxdbr0" +UPDATE_PROFILE="true" +LXD_CONFILE="" +LXD_DOMAIN="lxd" +LXD_IPV4_ADDR="10.44.127.1" +LXD_IPV4_NETMASK="255.255.255.0" +LXD_IPV4_NETWORK="10.44.127.1/24" +LXD_IPV4_DHCP_RANGE="10.44.127.2,10.44.127.254" +LXD_IPV4_DHCP_MAX="252" +LXD_IPV4_NAT="true" +LXD_IPV6_ADDR="" +LXD_IPV6_MASK="" +LXD_IPV6_NETWORK="" +LXD_IPV6_NAT="false" +LXD_IPV6_PROXY="false" +EOF + +systemctl enable lxd-bridge +systemctl start lxd-bridge + +DEFAULT_INTERFACE=$(route -n | awk '$1~/^0.0.0.0/ {print $8}') +DEFAULT_MTU=$( ip addr show $DEFAULT_INTERFACE | perl -ne 'if (/mtu\s(\d+)/) {print $1;}') + +INFO "Setting lxdbr0 MTU to $DEFAULT_MTU" +ifconfig lxdbr0 mtu $DEFAULT_MTU + +# Make the MTU change persistent between reboots +sed -i '/ifconfig lxdbr0 mtu/d' /etc/rc.local +sed -i "$ i ifconfig lxdbr0 mtu $DEFAULT_MTU" /etc/rc.local + +INFO "Pre-caching Ubuntu:16.04 image (this may take several minutes)..." + +# Setup a daily cron to update the cached image +cp $HERE/update-lxd-image.sh /etc/cron.daily + +# Run it for the first time +/etc/cron.daily/update-lxd-image.sh xenial + +INFO "Bootstrapping VCA" +juju bootstrap localhost osm \ +--config default-series=xenial \ +--config enable-os-refresh-update=false \ +--config enable-os-upgrade=false \ +--config apt-http-proxy=http://10.44.127.1:8000 + +RC=0 + +INFO "done, RC=$RC" +exit $RC diff --git a/jenkins/VCA/update-lxd-image.sh b/jenkins/VCA/update-lxd-image.sh new file mode 100755 index 00000000..182a33c2 --- /dev/null +++ b/jenkins/VCA/update-lxd-image.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# This script will create xenial and trusty lxd images that will be used by the +# lxd provider in juju 2.1+ It is for use with the lxd provider for local +# development and preinstalls a common set of production packages. +# +# This dramatically speeds up the install hooks for lxd deploys by +# pre-installing common packages. It is intended to run daily as part of +# a cron job. +set -eux + +# The basic charm layer also installs all the things. 47 packages. +LAYER_BASIC="gcc build-essential python3-pip python3-setuptools python3-yaml" + +# the basic layer also installs virtualenv, but the name changed in xenial. +TRUSTY_PACKAGES="python-virtualenv" +XENIAL_PACKAGES="virtualenv" + +# Predownload common packages used by your charms in development +DOWNLOAD_PACKAGES="" + +PACKAGES="$LAYER_BASIC $DOWNLOAD_PACKAGES" + +function cache() { + series=$1 + container=juju-${series}-base + alias=juju/$series/amd64 + + lxc delete $container -f || true + lxc launch ubuntu:$series $container + + # Wait for the container to get an IP address + lxc exec $container -- bash -c "for i in {1..60}; do sleep 1; ping -c1 10.44.127.1 &> /dev/null && break; done" + + lxc exec $container -- apt-get update -y + lxc exec $container -- apt-get upgrade -y + lxc exec $container -- apt-get install -y $PACKAGES $2 + lxc stop $container + + lxc image delete $alias || true + lxc publish $container --alias $alias description="$series juju dev image ($(date +%Y%m%d))" + + lxc delete $container -f || true +} + +cache trusty "$TRUSTY_PACKAGES" +cache xenial "$XENIAL_PACKAGES" diff --git a/jenkins/common/all_funcs b/jenkins/common/all_funcs new file mode 100644 index 00000000..a8471087 --- /dev/null +++ b/jenkins/common/all_funcs @@ -0,0 +1,28 @@ +# this file is meant to be sourced +# +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# import all functions +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis + +if [ -z "$OSM_JENKINS" ]; then + export OSM_JENKINS=$(realpath $(dirname ${BASH_SOURCE[0]} )) +fi + +for file in logging config container git_functions; do + . ${OSM_JENKINS}/common/$file + INFO "$file sourced" +done diff --git a/jenkins/common/config b/jenkins/common/config new file mode 100644 index 00000000..7f49381e --- /dev/null +++ b/jenkins/common/config @@ -0,0 +1,40 @@ +#!/bin/bash +# This file is meant to be SOURCED +# +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# config functions +# 24 June 2016 -- Jeremy Mordkoff -- Genesis + + +OSM_load_config() { + OSM_load_config_file ${OSM_JENKINS}/SETTINGS + if [ -z "$OSM_MDG" ]; then + WARNING "OSM_MDG not set" + else + OSM_load_config_file ${OSM_JENKINS}/${OSM_MDG}/SETTINGS + fi +} + +OSM_load_config_file() { + [ $# -eq 1 ] || FATAL "arg is filename" + if [ -f "$1" ]; then + . "$1" + INFO "config file $1 loaded" + else + WARNING "$1 not found" + fi +} + diff --git a/jenkins/common/container b/jenkins/common/container new file mode 100644 index 00000000..e29d5eb3 --- /dev/null +++ b/jenkins/common/container @@ -0,0 +1,89 @@ +# This file is meant to be SOURCED +# +# Copyright 2016 RIFT.IO Inc +# Copyright 2016 Telefónica Investigación y Desarrollo S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# container_funcs +# 24 June 2016 -- Jeremy Mordkoff -- Genesis +# -- Gerardo García + +container_exists() { + if [ $# -ne 1 ]; then + FATAL "arg is container name" + fi + lxc config show $1 >/dev/null 2>&1 + if [ $? -eq 0 ]; then + DEBUG "container $1 exists" + return 0 + else + DEBUG "container $1 not found" + return 1 + fi +} + +create_container() { + if [ $# -lt 2 ]; then + FATAL "args are image container [options]" + fi + INFO "creating container $2 using image $1" + image=$1 + container=$2 + shift 2 + DEBUG "lxc launch $image $container $*" + lxc launch "$image" "$container" $* +} + +container_exec() { + container="$1" + shift + DEBUG "exec in $container \"$*\"" + lxc exec "$container" -- $* +} + +container_exec_stderr() { + container="$1" + shift + DEBUG "exec in $container \"$*\"" + lxc exec "$container" -- $* 2>&1 +} + +wait_container_up() { + [ $# -eq 1 ] || FATAL "arg is container name got $# args - $*" + RE="200" + ct=0 + while [ $ct -lt 10 ]; do + let ct=ct+1 + output=$(container_exec_stderr "$1" curl -sL -w "%{http_code}\\n" "http://www.google.com/" -o /dev/null) + if [[ $output =~ $RE ]]; then + DEBUG "$1 is up" + return + fi + INFO "waiting for container $1 to start" + DEBUG "expected '$RE' in $output" + sleep 5 + done + FATAL "container $1 did not start" +} +container_push_tree() { + # create a tarball locally, pipe it into the container and unpack it there + [ $# -eq 3 ] || FATAL "args are container dir_from dir_to (dir_to MUST exist)" + tar -C "$2" -c . -f - | container_exec $1 tar -C "$3" -x -f - +} + +container_push_devops() { + [ $# -eq 1 ] || FATAL "arg is container name got $# args - $*" + container_exec "$1" mkdir -p /root/devops + container_push_tree "$1" "$(dirname $OSM_JENKINS)" "/root/devops" +} diff --git a/jenkins/common/git_functions b/jenkins/common/git_functions new file mode 100644 index 00000000..e5b79845 --- /dev/null +++ b/jenkins/common/git_functions @@ -0,0 +1,40 @@ +#!/bin/bash + + +GIT() { + CMD git "$@" +} + + +OSM_git_checkout() { + + # Updates all the branches in the local repo (clones if it does not exist) + if [ -d $OSM_MDG ]; then + INFO "reusing existing workspace" + cd $OSM_MDG + GIT fetch --all --tags + #git checkout master #to make sure that we are in the right branch before pulling the code + #git pull + else + INFO "cloning MDG $OSM_MDG from $OSM_GIT_URL/$OSM_MDG" + GIT clone $OSM_GIT_URL/$OSM_MDG + cd $OSM_MDG + for remote in `git branch -r |grep -v /HEAD`; do GIT branch --track ${remote#origin/} $remote; done + fi + + if [ $# -gt 0 ]; then + if [ "$1" = "checkout" ]; then + INFO "Code to compile: '$2'" + GIT checkout $2 + else + INFO "Code to compile: gerrit refspec '$1', commit-id: '$2'" + GIT fetch origin $1 || FATAL "git fetch origin '$1' didn't work" + GIT checkout -f $2 || FATAL "git checkout -f '$2' didn't work" + fi + else + INFO "Code to compile: master" + GIT checkout master + fi + +} + diff --git a/jenkins/common/install_common b/jenkins/common/install_common new file mode 100755 index 00000000..d844bdd9 --- /dev/null +++ b/jenkins/common/install_common @@ -0,0 +1,48 @@ +# this file is meant to be sourced +# +# Copyright 2017 Sandvine +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +RELEASE="ReleaseTWO" +REPOSITORY_KEY="OSM%20ETSI%20Release%20Key.gpg" +REPOSITORY="stable" +REPOSITORY_BASE="http://osm-download.etsi.org/repository/osm/debian" + +while getopts ":r:k:u:R:" o; do + case "${o}" in + r) + REPOSITORY=${OPTARG} + ;; + R) + RELEASE=${OPTARG} + ;; + k) + REPOSITORY_KEY=${OPTARG} + ;; + u) + REPOSITORY_BASE=${OPTARG} + ;; + *) + exit 1 + ;; + esac +done + +key_location=$REPOSITORY_BASE/$RELEASE/$REPOSITORY_KEY + +curl $key_location | apt-key add - + +apt update && add-apt-repository -y "deb $REPOSITORY_BASE/$RELEASE $REPOSITORY SO UI RO osmclient openvim" diff --git a/jenkins/common/logging b/jenkins/common/logging new file mode 100644 index 00000000..a95b5633 --- /dev/null +++ b/jenkins/common/logging @@ -0,0 +1,61 @@ +#!/bin/bash +# This file is meant to be SOURCED +# +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# container_funcs +# 24 June 2016 -- Jeremy Mordkoff -- Genesis + +print_stack() { + local i + local stack_size=${#FUNCNAME[1]} + echo "BACKTRACE:" >&2 + for (( i=1; i<$stack_size ; i++ )); do + local func="${FUNCNAME[$i]}" + [ x$func = x ] && func=MAIN + local linen="${BASH_LINENO[(( i - 1 ))]}" + local src="${BASH_SOURCE[$i]}" + [ x"$src" = x ] && src=non_file_source + echo "### $func $src $linen" >&2 + done + echo "-------" >&2 +} + + +FATAL() { + echo -e "\n### $(date) ${FUNCNAME[1]}: FATAL error: $*" >&2 + print_stack + exit 1 +} + +WARNING() { + echo -e "\n### $(date) ${FUNCNAME[1]}: WARNING error: $*" >&2 +} + +INFO() { + echo "## $(date) ${FUNCNAME[1]}: $*" >&2 +} + +DEBUG() { + echo "# $(date) ${FUNCNAME[1]}: $*" >&2 +} + +CMD() { + echo "# executing '$*' ..." + "$@" + rc=$? + echo "# .... '$*' done RC was $rc" + return $rc +} diff --git a/jenkins/host/clean_container b/jenkins/host/clean_container new file mode 100755 index 00000000..601e6b86 --- /dev/null +++ b/jenkins/host/clean_container @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2016 Telefónica Investigación y Desarrollo S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# clean_container is run on a host to clean an MDG container +# +# 20 Sept 2016 -- Gerardo Garcia -- Genesis +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +[ $# -lt 1 ] && FATAL "arg is MDG name" + +export OSM_MDG=$1 +OSM_load_config + +if container_exists $OSM_BUILD_CONTAINER; then + INFO "Container exists. Deleting ..." + lxc stop $OSM_BUILD_CONTAINER + lxc delete $OSM_BUILD_CONTAINER + INFO "$OSM_BUILD_CONTAINER container deleted." +else + INFO "Container does not exist. Nothing to be done." +fi + +INFO "$OSM_MDG clean-container DONE." +exit 0 + diff --git a/jenkins/host/install b/jenkins/host/install new file mode 100755 index 00000000..dfdbe596 --- /dev/null +++ b/jenkins/host/install @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# install is run on a host to install a MDG from binaries +# + + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +[ $# -lt 1 ] && FATAL "arg is MDG name" + +export OSM_MDG=$1 +shift +OSM_load_config + +if ! container_exists $OSM_BUILD_CONTAINER; then + CONTAINER_OPTS="" + [[ "$OSM_BUILD_CONTAINER_PRIVILEGED" == yes ]] && CONTAINER_OPTS="$CONTAINER_OPTS -c security.privileged=true" + [[ "$OSM_BUILD_CONTAINER_ALLOW_NESTED" == yes ]] && CONTAINER_OPTS="$CONTAINER_OPTS -c security.nesting=true" + create_container $OSM_BASE_IMAGE $OSM_BUILD_CONTAINER $CONTAINER_OPTS + wait_container_up $OSM_BUILD_CONTAINER + if [ ${OSM_USE_LOCAL_DEVOPS:-false} != false ]; then + container_push_devops $OSM_BUILD_CONTAINER + else + container_exec $OSM_BUILD_CONTAINER git clone ${OSM_GIT_URL}/devops + fi +else + if [ ${OSM_USE_LOCAL_DEVOPS:-false} != false ]; then + container_push_devops $OSM_BUILD_CONTAINER + else + container_exec $OSM_BUILD_CONTAINER git -C devops pull + fi +fi + +container_exec $OSM_BUILD_CONTAINER ./devops/jenkins/$OSM_MDG/install $* +RC=$? +INFO "$OSM_MDG install complete. Return code was $RC" +exit $RC + diff --git a/jenkins/host/start_build b/jenkins/host/start_build new file mode 100755 index 00000000..02bfcc26 --- /dev/null +++ b/jenkins/host/start_build @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright 2016 RIFT.IO Inc +# Copyright 2016 Telefónica Investigación y Desarrollo S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# start-build is run on a host to start a MDG build +# +# 24 June 2016 -- Jeremy Mordkoff -- Genesis +# -- Gerardo Garcia + + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +[ $# -lt 1 ] && FATAL "arg is MDG name" + +export OSM_MDG=$1 +shift +OSM_load_config + +if [ "$1" = "--build-container" ]; then + shift + [ $# -lt 1 ] && FATAL "missing container name with option --build-container" + export OSM_BUILD_CONTAINER=$1 + shift +fi + +if ! container_exists $OSM_BUILD_CONTAINER; then + CONTAINER_OPTS="" + [[ "$OSM_BUILD_CONTAINER_PRIVILEGED" == yes ]] && CONTAINER_OPTS="$CONTAINER_OPTS -c security.privileged=true" + [[ "$OSM_BUILD_CONTAINER_ALLOW_NESTED" == yes ]] && CONTAINER_OPTS="$CONTAINER_OPTS -c security.nesting=true" + [[ "$OSM_BUILD_CONTAINER_ALLOW_DOCKER" == yes ]] && CONTAINER_OPTS="$CONTAINER_OPTS -p docker -p default" + create_container $OSM_BASE_IMAGE $OSM_BUILD_CONTAINER $CONTAINER_OPTS + wait_container_up $OSM_BUILD_CONTAINER + RE="fedora|fc[0-9]" + if [[ $OSM_BASE_IMAGE =~ $RE ]]; then + container_exec $OSM_BUILD_CONTAINER yum -y install git tar make sudo + else + container_exec $OSM_BUILD_CONTAINER apt -y install git realpath make sudo + fi + if [ ${OSM_USE_LOCAL_DEVOPS:-false} ]; then + container_push_devops $OSM_BUILD_CONTAINER + else + container_exec $OSM_BUILD_CONTAINER git clone ${OSM_GIT_URL}/devops + fi +else + if [ ${OSM_USE_LOCAL_DEVOPS:-false} ]; then + container_push_devops $OSM_BUILD_CONTAINER + else + container_exec $OSM_BUILD_CONTAINER git -C devops pull + fi +fi + +container_exec $OSM_BUILD_CONTAINER ./devops/jenkins/$OSM_MDG/start_build $* +RC=$? +INFO "$OSM_MDG build complete. Return code was $RC" +exit $RC + diff --git a/jenkins/osmclient/SETTINGS b/jenkins/osmclient/SETTINGS new file mode 100644 index 00000000..36e50151 --- /dev/null +++ b/jenkins/osmclient/SETTINGS @@ -0,0 +1,28 @@ +# +# Copyright 2017 Sandvine +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# this variable holds the name of the container image needed to build or run this product +export OSM_BASE_IMAGE=ubuntu:16.04 +# +# this variable holds the name of the container to be used to build a package +# if this container already exists, the build process can skip the container build +export OSM_BUILD_CONTAINER=osmclient +# +# this variable holds the name of the container to be used to run a package +# if this container already exists, the run processes can skip the container build +export OSM_RUNTIME_CONTAINER=osmclient + +export OSM_BUILD_CONTAINER_ALLOW_DOCKER=yes diff --git a/jenkins/osmclient/start_build b/jenkins/osmclient/start_build new file mode 100755 index 00000000..2206a1a7 --- /dev/null +++ b/jenkins/osmclient/start_build @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Copyright 2017 Sandvine +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Michael Marchetti - mmarchetti@sandvine.com +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +# SET YOU MDG repository name here +export OSM_MDG=osmclient +OSM_load_config + +NOTEST="" +if [ "$1" = "--notest" ]; then + shift + NOTEST="y" +fi + +OSM_git_checkout "$@" + +apt update +apt install -y docker.io + +# cleanup all pre-existing builds. +# deb pkg build fails if previous build image is present +rm -rf deb_dist dist + +docker build -t $OSM_MDG . +DOCKER_ARGS="-v $(pwd):$(pwd) -w $(pwd) -u $(id -u):$(id -g) $OSM_MDG" + +if [ -n "$NOTEST" ]; then + docker run $DOCKER_ARGS /bin/bash -c "./docker_command.sh $(id -u -n) $(id -g -n) tox" +fi + +docker run $DOCKER_ARGS /bin/bash -c "./docker_command.sh $(id -u -n) $(id -g -n) tox -e build" + +RC=$? +INFO "done, RC=$RC" +exit $RC diff --git a/jenkins/release/delete-tag.sh b/jenkins/release/delete-tag.sh new file mode 100755 index 00000000..358d6806 --- /dev/null +++ b/jenkins/release/delete-tag.sh @@ -0,0 +1,25 @@ +#!/bin/bash +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +echo $OSM_JENKINS +. $OSM_JENKINS/common/all_funcs + +[ $# -ne 1 ] && FATAL "arg1 is tag to be deleted" + +TAG="$1" + +TEMPDIR="$(mktemp -q -d --tmpdir "tagosm.XXXXXX")" +trap 'rm -rf "$TEMPDIR"' EXIT + +list="juju-charms devops descriptor-packages openvim RO SO UI" +for i in $list; do + REPO_FOLDER="$TEMPDIR/$i" + echo + echo "Cloning $i" + #git -C $TEMPDIR clone ssh://garciadeblas@osm.etsi.org:29418/osm/$i + git -C $REPO_FOLDER tag -d $TAG + git -C $REPO_FOLDER push origin :refs/tags/$TAG + sleep 2 + rm -rf $REPO_FOLDER +done + diff --git a/jenkins/release/new-tag.sh b/jenkins/release/new-tag.sh new file mode 100755 index 00000000..37c5193b --- /dev/null +++ b/jenkins/release/new-tag.sh @@ -0,0 +1,48 @@ +#!/bin/bash +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +echo $OSM_JENKINS +. $OSM_JENKINS/common/all_funcs + +[ $# -ne 2 ] && FATAL "arg1 is branch, arg2 is new tag" + +#CURRENT_BRANCH="v1.1" +#TAG="v1.1.0" +CURRENT_BRANCH="$1" +TAG="$2" + +#tag_header="OSM Release ONE:" +tag_header="OSM" +tag_message="$tag_header version $TAG" + +TEMPDIR="$(mktemp -q -d --tmpdir "tagosm.XXXXXX")" +trap 'rm -rf "$TEMPDIR"' EXIT +#chmod 0600 "$TEMPDIR" + +#juju-charms and devops repos have no vx.y branch yet +list="juju-charms devops" +for i in $list; do + REPO_FOLDER="$TEMPDIR/$i" + echo + echo "Cloning and tagging $i" + #git -C $TEMPDIR clone ssh://garciadeblas@osm.etsi.org:29418/osm/$i + git -C $REPO_FOLDER checkout master + git -C $REPO_FOLDER tag -a $TAG -m"$tag_message" + git -C $REPO_FOLDER push origin $TAG --follow-tags + sleep 2 + rm -rf $REPO_FOLDER +done + +list="descriptor-packages openvim RO SO UI" +for i in $list; do + REPO_FOLDER="$TEMPDIR/$i" + echo + echo "Cloning and tagging $i" + #git -C $TEMPDIR clone ssh://garciadeblas@osm.etsi.org:29418/osm/$i + git -C $REPO_FOLDER checkout $CURRENT_BRANCH + git -C $REPO_FOLDER tag -a $TAG -m"$tag_message" + git -C $REPO_FOLDER push origin $TAG --follow-tags + sleep 2 + rm -rf $REPO_FOLDER +done + diff --git a/jenkins/system/Jenkinsfile b/jenkins/system/Jenkinsfile new file mode 100644 index 00000000..8a04f236 --- /dev/null +++ b/jenkins/system/Jenkinsfile @@ -0,0 +1,85 @@ +// input parameters: +// boolean: BUILD_FROM_SOURCE +// boolean: REPO_DISTRO +// boolean: COMMIT_ID +// boolean: UPSTREAM_SUFFIX +// string: NODE +// string: RSYNC_DESTINATION +// string: REPO_BASE_URL +// string: REPO_KEY_NAME +// string: RELEASE + +node("${params.NODE}") { + + stage("Setup") { + tag_or_branch = params.COMMIT_ID.replaceAll(/\./,"") + container_name_prefix = "osm-${tag_or_branch}" + container_name = "${container_name_prefix}-${BUILD_NUMBER}" + } + + stage("Checkout") { + checkout scm + } + + // Copy the artifacts from the upstream jobs + stage("Copy Artifacts") { + // cleanup any previous repo + sh 'rm -rf repo' + + dir('repo') { + // grab all stable upstream builds based on the + // given target UPSTREAM_SUFFIX + + def list = ["SO", "UI", "RO", "openvim", "osmclient"] + for (component in list) { + step ([$class: 'CopyArtifact', + projectName: "${component}_${params.UPSTREAM_SUFFIX}"]) + sh "dpkg-sig --sign builder -k dpkg1 pool/${component}/*" + // cleanup any prevously defined dists + sh "rm -rf dists" + } + + // now create the distro + for (component in list) { + sh "mkdir -p dists/${params.REPO_DISTRO}/${component}/binary-amd64/" + sh "apt-ftparchive packages pool/${component} > dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages" + sh "gzip -9fk dists/${params.REPO_DISTRO}/${component}/binary-amd64/Packages" + } + + // create and sign the release file + sh "apt-ftparchive release dists/${params.REPO_DISTRO} > dists/${params.REPO_DISTRO}/Release" + sh "gpg --yes -abs -u dpkg1 -o dists/${params.REPO_DISTRO}/Release.gpg dists/${params.REPO_DISTRO}/Release" + sh "rsync -avz . ${params.RSYNC_DESTINATION}/${params.RELEASE}" + } + } + + stage("Cleanup") { + // check for previous containers and clean them up + sh "jenkins/system/delete_old_containers.sh ${container_name_prefix}" + } + + stage("Build") { + from_source = '' + if ( params.BUILD_FROM_SOURCE ) + { + from_source = '--source' + } + + sh """ + export OSM_USE_LOCAL_DEVOPS=true + jenkins/host/start_build system --build-container ${container_name} \ + -b ${params.COMMIT_ID} \ + -r ${params.REPO_DISTRO} \ + -u ${params.REPO_BASE_URL} \ + -k ${params.REPO_KEY_NAME} \ + -R ${params.RELEASE} \ + ${from_source} + """ + } + + stage("Archive Artifacts") { + sh "echo ${container_name} > build_version.txt" + sh "tar -zcvf repo.tar.gz repo" + archiveArtifacts artifacts: "build_version.txt, repo.tar.gz", fingerprint: true + } +} diff --git a/jenkins/system/SETTINGS b/jenkins/system/SETTINGS new file mode 100644 index 00000000..912799c6 --- /dev/null +++ b/jenkins/system/SETTINGS @@ -0,0 +1,39 @@ +# Copyright 2016 Telefónica Investigación y Desarrollo, S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# sample SETTINGS file +# +# Authors: +# - Gerardo Garcia +# +# this variable holds the name of the container image needed to build or run this product +export OSM_BASE_IMAGE=ubuntu:16.04 +# +# this variable must be set to allow creating the build container in privileged mode +# this variable should be removed in the future when no privileged mode is required +export OSM_BUILD_CONTAINER_PRIVILEGED=yes +# +# this variable must be set to allow ensted containers in the build container +export OSM_BUILD_CONTAINER_ALLOW_NESTED=yes +# +# this variable must be set to allow creating the runtime container in privileged mode +# this variable should be removed in the future when no privileged mode is required +export OSM_RUNTIME_CONTAINER_PRIVILEGED=yes +# +# this variable must be set to allow ensted containers in the build container +export OSM_RUNTIME_CONTAINER_ALLOW_NESTED=yes +# +# +export OSM_BUILD_CONTAINER=osm +export OSM_RUNTIME_CONTAINER=osm diff --git a/jenkins/system/delete_old_containers.sh b/jenkins/system/delete_old_containers.sh new file mode 100755 index 00000000..13c6df42 --- /dev/null +++ b/jenkins/system/delete_old_containers.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2017 Sandvine +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A helper routine that cleans up old container images based +# on an input prefix to check. Jenkins builds will add an incrementing +# build suffix (build number) to the prefix +# +#$1 container prefix name + +keep_number=1 +prefix=$1 + +# keep the first build +keep=$(lxc list | grep $prefix | awk '{print $2}' | sort -rn | head -n$keep_number) +for container in $(lxc list | grep $prefix | awk '{print $2}' | sort -rn); do + if [ "$container" != "$keep" ]; then + echo "deleting old container $container" + lxc delete $container --force + fi +done diff --git a/jenkins/system/start_build b/jenkins/system/start_build new file mode 100755 index 00000000..7f259c72 --- /dev/null +++ b/jenkins/system/start_build @@ -0,0 +1,78 @@ +#!/bin/bash +# Copyright 2017 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# 01 May 2017 -- Michael Marchetti -- adapted from template +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +INFO "Installing packages" +apt-get update + +INFO "Configuring LXD" +# ZFS doesn't work inside a nested container. ZFS should be configured in the host LXD. +lxd init --auto +lxd waitready +systemctl stop lxd-bridge +systemctl --system daemon-reload +cat < /etc/default/lxd-bridge +USE_LXD_BRIDGE="true" +LXD_BRIDGE="lxdbr0" +UPDATE_PROFILE="true" +LXD_CONFILE="" +LXD_DOMAIN="lxd" +LXD_IPV4_ADDR="10.44.126.1" +LXD_IPV4_NETMASK="255.255.255.0" +LXD_IPV4_NETWORK="10.44.126.1/24" +LXD_IPV4_DHCP_RANGE="10.44.126.2,10.44.126.254" +LXD_IPV4_DHCP_MAX="252" +LXD_IPV4_NAT="true" +LXD_IPV6_ADDR="" +LXD_IPV6_MASK="" +LXD_IPV6_NETWORK="" +LXD_IPV6_NAT="false" +LXD_IPV6_PROXY="false" +EOF + +systemctl enable lxd-bridge +systemctl start lxd-bridge + +apt-get install -y python-pip python python-pycurl charm-tools python-pytest + +# TODO: use package when available on osm repo +git clone https://osm.etsi.org/gerrit/osm/osmclient +pip install osmclient/. + +export OSM_USE_LOCAL_DEVOPS=true +devops/installers/install_osm.sh --test $* +RC=$? + +# workaround. for upload packages, lxdbr0 needs to be promiscuous +# as the upload calls back to the UI server so the lxdbr0 needs +# to operate as a bridge +ifconfig lxdbr0 promisc + +if [ $RC == 0 ]; then + # success. find all the resulting containers + . devops/installers/export_ips + + TO_ADD="export OSM_HOSTNAME=$SO_CONTAINER_IP" + grep -q OSM_HOSTNAME ~/.bashrc && sed -i "s/.*OSM_HOSTNAME.*/$TO_ADD/" ~/.bashrc || echo -e "$TO_ADD\n$(cat ~/.bashrc)" > ~/.bashrc +fi + +INFO "done, RC=$RC" +exit $RC diff --git a/jenkins/template/SETTINGS b/jenkins/template/SETTINGS new file mode 100644 index 00000000..94154f3b --- /dev/null +++ b/jenkins/template/SETTINGS @@ -0,0 +1,46 @@ +# Copyright 2016 RIFT.IO Inc +# Copyright 2016 Telefónica Investigación y Desarrollo, S.A.U. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# sample SETTINGS file +# +# Authors: +# 24 June 2016 -- Jeremy Mordkoff -- Genesis +# -- Gerardo Garcia +# +# this variable holds the name of the container image needed to build or run this product +export OSM_BASE_IMAGE=ubuntu:16.04 +# +# this variable holds the name of the container to be used to build a package +# if this container already exists, the build process can skip the container build +export OSM_BUILD_CONTAINER=container_name-build +# +# this variable must be set to allow creating the build container in privileged mode +# this variable should be removed in the future when no privileged mode is required +#export OSM_BUILD_CONTAINER_PRIVILEGED=yes +# +# this variable must be set to allow ensted containers in the build container +#export OSM_BUILD_CONTAINER_ALLOW_NESTED=yes +# +# this variable holds the name of the container to be used to run a package +# if this container already exists, the run processes can skip the container build +export OSM_RUNTIME_CONTAINER=container_name-runtime +# +# this variable must be set to allow creating the runtime container in privileged mode +# this variable should be removed in the future when no privileged mode is required +#export OSM_RUNTIME_CONTAINER_PRIVILEGED=yes +# +# this variable must be set to allow ensted containers in the build container +#export OSM_RUNTIME_CONTAINER_ALLOW_NESTED=yes +# diff --git a/jenkins/template/start_build b/jenkins/template/start_build new file mode 100755 index 00000000..43ef1610 --- /dev/null +++ b/jenkins/template/start_build @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright 2016 RIFT.IO Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# TEMPLATE script to start a build. This is run inside a container +# +# 6 July 2016 -- Jeremy.Mordkoff@riftio.com -- adapted from the riftware version +# + +HERE=$(realpath $(dirname $0)) +OSM_JENKINS=$(dirname $HERE) +. $OSM_JENKINS/common/all_funcs + +# SET YOU MDG repository name here +export OSM_MDG=XXXX +OSM_load_config +OSM_git_checkout "$@" + +INFO "starting build" +### for start_build +### put your commands here to +### build, test and produce coverage reports +# E.G. +#make clean || FATAL "make clean failed" +#make || FATAL "make failed" +#sudo make install || FATAL "make install failed" + +RC=0 + +INFO "done, RC=$RC" +exit $RC + + diff --git a/juju-charms/Makefile b/juju-charms/Makefile new file mode 100644 index 00000000..a579390f --- /dev/null +++ b/juju-charms/Makefile @@ -0,0 +1,33 @@ +# +# Copyright 2016 RIFT.io Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +BUILD_DIR = . + +CHARMS:= vpe-router vyos-proxy pingpong flownac sandvine-pts-proxy +CHARM_SRC_DIR := layers +CHARM_BUILD_DIR := $(BUILD_DIR)/builds + +CHARM_SRC_DIRS := $(addprefix $(CHARM_SRC_DIR)/, $(CHARMS)) +CHARM_BUILD_DIRS := $(addprefix $(CHARM_BUILD_DIR)/, $(CHARMS)) + +all: $(CHARM_BUILD_DIRS) + +clean: + -@ $(RM) -rf $(CHARM_BUILD_DIR) + +$(CHARM_BUILD_DIR)/%: $(CHARM_SRC_DIR)/% + charm-build -o $(BUILD_DIR) $< diff --git a/juju-charms/README.md b/juju-charms/README.md new file mode 100644 index 00000000..6eaf43f8 --- /dev/null +++ b/juju-charms/README.md @@ -0,0 +1,79 @@ +# Juju Charm usage and development + +This document is intended to provide a brief overview of the components included +in this repository as well as recommendations for how to develop, build, and +publish charms. + +Please read the [develper geting started guide](https://jujucharms.com/docs/2.0/developer-getting-started) before proceeding. + +## Directory structure + +``` +. +├── builds +│   └── vpe-router +├── interfaces +├── layers +│   └── vpe-router +└── module-blueprints +``` + +The source code of a charm is referred to as a "layer". This layer is compiled +into a charm and placed in the `builds/` directory. Interfaces, currently +unused in this context, extend relationships between applications. + +## Development workflow +### Prepare your build environment +``` +# Source the environment variables JUJU_REPOSITORY, INTERFACE_PATH, and +# LAYER_PATH, which are needed to build a charm. You could also place these +# in your $HOME/.bashrc +$ source juju-env.sh +``` +#### Install the `charm` command, either via apt: + +``` +$ sudo apt install charm +``` + +or with [snap](http://snapcraft.io/) + +``` +$ snap install charm --edge +``` + +To build a charm, simply run `charm build` inside of a layer. +``` +$ cd $LAYER_PATH/vpe-router +$ charm build +$ charm deploy $JUJU_REPOSITORY/builds/vpe-router +``` + +## Publishing to jujucharms.com + +Publishing to the Juju Charm store requires a launchpad login. With that, login +to [jujucharms.com](http://www.jujucharms.com/). + +Next, you'll use the charm command to publish your compiled charm. This will +put the charm into the store where it can be used by anyone with access. + +For example, if I wanted to publish the latest version of the vpe-router charm: + +# Step 1: Upload the charm to the "unpublished" channel +``` +$ cd $JUJU_REPOSITORY/builds +$ charm push vpe-router/ cs:~aisrael/vpe-router +url: cs:~aisrael/vpe-router-0 +channel: unpublished +``` + +# There are four channels to release a charm: edge, beta, candidate, and stable +``` +$ charm release cs:~aisrael/vpe-router-0 --channel=edge +url: cs:~aisrael/vpe-router-0 +channel: edge +``` +The charm can then be deployed directly from the charm store: +``` +$ juju deploy cs:~aisrael/vpe-router --channel=edge +``` diff --git a/juju-charms/juju-env.sh b/juju-charms/juju-env.sh new file mode 100644 index 00000000..59fa9e71 --- /dev/null +++ b/juju-charms/juju-env.sh @@ -0,0 +1,4 @@ +# Set the Juju env variables for building a layer +export JUJU_REPOSITORY=`pwd` +export INTERFACE_PATH=$JUJU_REPOSITORY/interfaces +export LAYER_PATH=$JUJU_REPOSITORY/layers diff --git a/juju-charms/layers/netutils/LICENSE b/juju-charms/layers/netutils/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/juju-charms/layers/netutils/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/juju-charms/layers/netutils/README.md b/juju-charms/layers/netutils/README.md new file mode 100644 index 00000000..e8258c03 --- /dev/null +++ b/juju-charms/layers/netutils/README.md @@ -0,0 +1,64 @@ +# Overview + +This charm provides basic network utilities that can be run from a Juju-deployed +machine. + +# Usage + +To deploy the charm: +```bash +$ juju deploy cs:~nfv/netutils +``` + +To run an action: +```bash +$ juju run-action netutils/0 ping destination=google.com +$ juju run-action netutils/0 traceroute destination=google.com +``` + +To fetch the output of an action: +```bash +$ juju show-action-output 026b3d4c-0bb2-4818-8d24-9855936cdcdf +results: + output: | + traceroute to google.com (216.58.198.78), 30 hops max, 60 byte packets + 1 ec2-79-125-0-86.eu-west-1.compute.amazonaws.com (79.125.0.86) 1.431 ms 1.410 ms 1.380 ms + 2 100.64.2.73 (100.64.2.73) 1.647 ms 100.64.2.103 (100.64.2.103) 1.247 ms 100.64.2.121 (100.64.2.121) 1.224 ms + 3 100.64.0.232 (100.64.0.232) 1.296 ms 100.64.0.184 (100.64.0.184) 1.515 ms 100.64.0.234 (100.64.0.234) 1.079 ms + 4 100.64.16.37 (100.64.16.37) 0.377 ms 100.64.16.49 (100.64.16.49) 0.347 ms 100.64.16.1 (100.64.16.1) 0.340 ms + 5 176.32.107.12 (176.32.107.12) 0.739 ms 176.32.107.4 (176.32.107.4) 0.875 ms 0.748 ms + 6 178.236.0.111 (178.236.0.111) 0.650 ms 0.641 ms 0.645 ms + 7 72.14.215.85 (72.14.215.85) 0.544 ms 1.508 ms 1.498 ms + 8 209.85.252.198 (209.85.252.198) 0.680 ms 0.659 ms 0.618 ms + 9 64.233.174.27 (64.233.174.27) 0.690 ms 0.682 ms 0.634 ms + 10 dub08s02-in-f14.1e100.net (216.58.198.78) 0.568 ms 0.560 ms 0.595 ms +status: completed +timing: + completed: 2016-06-29 14:50:04 +0000 UTC + enqueued: 2016-06-29 14:50:03 +0000 UTC + started: 2016-06-29 14:50:03 +0000 UTC +``` +## iperf3 + +Because iperf3 has a client and server component, the netutils charm can operate +as both. Setting the iperf3 configuration value to True will start iperf3 in +server mode, running as a daemon. +``` +$ juju deploy cs:~nfv/netutils client +$ juju deploy cs:~nfv/netutils server iperf3=True +$ juju run-action client/0 iperf host= [...] +``` + +## Scale out Usage + +With great scalability comes great power, but please don't use this to DDoS anyone without their permission. + +## Known Limitations and Issues + +# Contact Information + +## Contributing to the charm + + - The compiled charm can be found [here](https://www.jujucharms.com/u/nfv/netutils). + - [layer/netutils](https://osm.etsi.org/gitweb/?p=osm/juju-charms.git;a=summary/) contains the source of the layer. + - Please add any bugs or feature requests to the [bugzilla](https://osm.etsi.org/bugzilla/buglist.cgi?component=Juju-charms&list_id=426&product=OSM&resolution=---). diff --git a/juju-charms/layers/netutils/actions.yaml b/juju-charms/layers/netutils/actions.yaml new file mode 100644 index 00000000..f4f78843 --- /dev/null +++ b/juju-charms/layers/netutils/actions.yaml @@ -0,0 +1,133 @@ +nmap: + description: "nmap a thing!" + params: + destination: + description: "destination to scan" + type: string + required: + - destination +ping: + description: 'ping a thing!' + params: + count: + description: "Stop after sending count ECHO_REQUEST packets" + type: integer + default: 30 + destination: + description: "destination of ping request" + type: string + required: + - destination +traceroute: + description: 'trace a thing!' + params: + hops: + description: "Stop tracing after count hops" + type: integer + default: 30 + destination: + description: "destination of traceroute request" + type: string + required: + - destination +dig: + description: "DNS lookup" + params: + nsserver: + description: "The nameserver to lookup against." + type: string + host: + description: "The host to lookup" + type: string + type: + description: "The DNS record type to lookup" + type: string + required: + - host +iperf: + description: "" + params: + host: + description: "" + type: string + port: + description: "" + type: integer + default: 5201 + format: + description: "" + type: string + interval: + description: "" + type: string + affinity: + description: "" + type: string + udp: + description: "Use UDP rather than TCP" + type: boolean + default: False + bandwidth: + description: "Set the target bandwidth to n bits/sec (default 1Mbit/sec for UDP, unlimited for TCP)" + type: integer + default: 1 + time: + description: "Time, in seconds, to transmit for." + type: integer + default: 10 + blockcount: + description: "The number of blocks to transmit" + type: integer + length: + description: "The length of buffer to read or write (default 128KB for TCP, 8KB for UDP)" + type: integer + parallel: + description: "The number of parallel client streams to run" + type: integer + reverse: + description: "Run in reverse mode (server sends, client receives)." + type: boolean + default: false + window: + description: "Window size/socket buffer size." + type: integer + bind: + description: "Bind to a specific interface or multicast address" + type: string + mss: + description: "Set the TCP maximum segment size (MTU - 40 bytes)" + type: integer + no-delay: + description: "Set the TCP no delay, disabling Nagle's algorithm." + type: boolean + default: false + ipv4: + description: "Only use IPv4" + type: boolean + default: false + ipv6: + description: "Only use IPv6" + type: boolean + default: false + tos: + description: "Set the IP 'type of service'" + type: integer + flowlabel: + description: "Set the IPv6 flow label (linux-only)" + type: string + zerocopy: + description: "Use a 'zero copy' method of sending data, such as sendfile(s), instead of the usual write(2)." + type: boolean + default: false + omit: + description: "Omit the first n seconds of the test, to skip past the TCP slow-start period." + type: integer + title: + description: "Prefix every output line with this string." + type: string + congestion: + description: "Set the linux congestion control algorithm." + type: string + + required: + - host diff --git a/juju-charms/layers/netutils/actions/dig b/juju-charms/layers/netutils/actions/dig new file mode 100755 index 00000000..736a4069 --- /dev/null +++ b/juju-charms/layers/netutils/actions/dig @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.dig') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/netutils/actions/iperf b/juju-charms/layers/netutils/actions/iperf new file mode 100755 index 00000000..750028e0 --- /dev/null +++ b/juju-charms/layers/netutils/actions/iperf @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.iperf3') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/netutils/actions/nmap b/juju-charms/layers/netutils/actions/nmap new file mode 100755 index 00000000..ede4f5b9 --- /dev/null +++ b/juju-charms/layers/netutils/actions/nmap @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.nmap') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/netutils/actions/ping b/juju-charms/layers/netutils/actions/ping new file mode 100755 index 00000000..9850fe76 --- /dev/null +++ b/juju-charms/layers/netutils/actions/ping @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.ping') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/netutils/actions/traceroute b/juju-charms/layers/netutils/actions/traceroute new file mode 100755 index 00000000..229ed327 --- /dev/null +++ b/juju-charms/layers/netutils/actions/traceroute @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.traceroute') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/netutils/config.yaml b/juju-charms/layers/netutils/config.yaml new file mode 100644 index 00000000..61010636 --- /dev/null +++ b/juju-charms/layers/netutils/config.yaml @@ -0,0 +1,5 @@ +options: + iperf3: + type: boolean + default: false + description: "Enabling this option will start iperf3 in server mode." diff --git a/juju-charms/layers/netutils/icon.svg b/juju-charms/layers/netutils/icon.svg new file mode 100644 index 00000000..e092eef7 --- /dev/null +++ b/juju-charms/layers/netutils/icon.svg @@ -0,0 +1,279 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + diff --git a/juju-charms/layers/netutils/layer.yaml b/juju-charms/layers/netutils/layer.yaml new file mode 100644 index 00000000..e13c81a8 --- /dev/null +++ b/juju-charms/layers/netutils/layer.yaml @@ -0,0 +1,8 @@ +repo: git@github.com:AdamIsrael/layer-netutils.git +includes: ['layer:basic', 'layer:sshproxy'] +options: + basic: + packages: + - traceroute + - nmap + - iperf3 diff --git a/juju-charms/layers/netutils/metadata.yaml b/juju-charms/layers/netutils/metadata.yaml new file mode 100644 index 00000000..c42d9637 --- /dev/null +++ b/juju-charms/layers/netutils/metadata.yaml @@ -0,0 +1,15 @@ +name: netutils +summary: A suite of network-related utilities. +maintainer: Adam Israel +description: | + A suite of network-related utilities, such as ping and traceroute, that + can be deployed into a data center in order to diagnose connectivity issues. +tags: + # https://jujucharms.com/docs/stable/authors-charm-metadata + - ops + - network + - performance +series: + - trusty + - xenial +subordinate: false diff --git a/juju-charms/layers/netutils/reactive/layer_netutils.py b/juju-charms/layers/netutils/reactive/layer_netutils.py new file mode 100644 index 00000000..1fd4cb24 --- /dev/null +++ b/juju-charms/layers/netutils/reactive/layer_netutils.py @@ -0,0 +1,137 @@ +from charmhelpers.core.hookenv import ( + action_get, + action_fail, + action_set, + config, + log, + status_set, +) + +from charms.reactive import ( + remove_state as remove_flag, + set_state as set_flag, + when, + when_not, +) +import charms.sshproxy +from subprocess import CalledProcessError + + +@when_not('netutils.ready') +def ready(): + status_set('active', 'Ready!') + set_flag('netutils.ready') + + +@when('actions.dig') +def dig(): + err = '' + try: + nsserver = action_get('nsserver') + host = action_get('host') + nstype = action_get('type') + cmd = "dig" + + if nsserver: + cmd += " @{}".format(nsserver) + if host: + cmd += " {}".format(host) + else: + action_fail('Hostname required.') + if nstype: + cmd += " -t {}".format(nstype) + + result, err = charms.sshproxy._run(cmd) + except: + action_fail('dig command failed:' + err) + else: + action_set({'outout': result}) + finally: + remove_flag('actions.dig') + + +@when('actions.nmap') +def nmap(): + err = '' + try: + result, err = charms.sshproxy._run( + 'nmap {}'.format(action_get('destination')) + ) + except: + action_fail('nmap command failed:' + err) + else: + action_set({'outout': result}) + finally: + remove_flag('actions.nmap') + + +@when('actions.ping') +def ping(): + err = '' + try: + result, err = charms.sshproxy._run('ping -qc {} {}'.format( + action_get('count'), action_get('destination')) + ) + + except: + action_fail('ping command failed:' + err) + else: + # Here you can send results back from ping, if you had time to parse it + action_set({'output': result}) + finally: + remove_flag('actions.ping') + + +@when('actions.traceroute') +def traceroute(): + try: + result, err = charms.sshproxy._run( + 'traceroute -m {} {}'.format( + action_get('hops'), + action_get('destination') + ) + ) + except: + action_fail('traceroute command failed') + else: + # Here you can send results back from ping, if you had time to parse it + action_set({'output': result}) + finally: + remove_flag('actions.traceroute') + + +@when('actions.iperf3') +def iperf3(): + err = '' + try: + # TODO: read all the flags via action_get and build the + # proper command line to run iperf3 + host = action_get('host') + + cmd = 'iperf3 -c {} --json'.format(host) + result, err = charms.sshproxy._run(cmd) + except CalledProcessError as e: + action_fail('iperf3 command failed:' + e.output) + else: + action_set({'outout': result}) + finally: + remove_flag('actions.iperf3') + + +@when('config.changed') +def config_changed(): + """ Handle configuration changes """ + cfg = config() + if cfg.changed('iperf3'): + if cfg['iperf3']: + # start iperf in server + daemon mode + cmd = "iperf3 -s -D" + else: + cmd = "killall iperf3" + try: + charms.sshproxy._run(cmd) + log("iperf3 stopped.") + except CalledProcessError: + log("iperf3 not running.") + else: + log("iperf3 started.") diff --git a/juju-charms/layers/netutils/tests/00-setup b/juju-charms/layers/netutils/tests/00-setup new file mode 100755 index 00000000..f0616a56 --- /dev/null +++ b/juju-charms/layers/netutils/tests/00-setup @@ -0,0 +1,5 @@ +#!/bin/bash + +sudo add-apt-repository ppa:juju/stable -y +sudo apt-get update +sudo apt-get install amulet python-requests -y diff --git a/juju-charms/layers/netutils/tests/10-deploy b/juju-charms/layers/netutils/tests/10-deploy new file mode 100755 index 00000000..ef269cda --- /dev/null +++ b/juju-charms/layers/netutils/tests/10-deploy @@ -0,0 +1,31 @@ +#!/usr/bin/python3 + +import amulet +import requests +import unittest + + +class TestCharm(unittest.TestCase): + def setUp(self): + self.d = amulet.Deployment() + + self.d.add('layer-netutils') + self.d.expose('layer-netutils') + + self.d.setup(timeout=900) + self.d.sentry.wait() + + self.unit = self.d.sentry['layer-netutils'][0] + + def test_service(self): + # test we can access over http + page = requests.get('http://{}'.format(self.unit.info['public-address'])) + self.assertEqual(page.status_code, 200) + # Now you can use self.d.sentry[SERVICE][UNIT] to address each of the units and perform + # more in-depth steps. Each self.d.sentry[SERVICE][UNIT] has the following methods: + # - .info - An array of the information of that unit from Juju + # - .file(PATH) - Get the details of a file on that unit + # - .file_contents(PATH) - Get plain text output of PATH file from that unit + # - .directory(PATH) - Get details of directory + # - .directory_contents(PATH) - List files and folders in PATH on that unit + # - .relation(relation, service:rel) - Get relation data from return service diff --git a/juju-charms/layers/pingpong/README.md b/juju-charms/layers/pingpong/README.md new file mode 100644 index 00000000..3bec2433 --- /dev/null +++ b/juju-charms/layers/pingpong/README.md @@ -0,0 +1,163 @@ +# Overview + +This repository contains the [Juju] layer that represents a working example of a proxy charm. + +# What is a proxy charm? + +A proxy charm is a limited type of charm that does not interact with software running on the same host, such as controlling and configuring a remote device (a static VM image, a router/switch, etc.). It cannot take advantage of some of Juju's key features, such as [scaling], [relations], and [leadership]. + +Proxy charms are primarily a stop-gap, intended to prototype quickly, with the end goal being to develop it into a full-featured charm, which installs and executes code on the same machine as the charm is running. + +# Usage + +```bash +# Clone this repository +git clone https://osm.etsi.org/gerrit/osm/juju-charms +cd juju-charms + +# Setup environment variables +source juju-env.sh + +cd layers/pingpong +charm build + +# Examine the built charm +cd ../../builds/pingpong +ls +actions config.yaml icon.svg metadata.yaml tests +actions.yaml copyright layer.yaml reactive tox.ini +bin deps lib README.md wheelhouse +builds hooks Makefile requirements.txt + +``` + +You can view a screencast of this: https://asciinema.org/a/96738 + +The `charm build` process combines this pingpong layer with each layer that it +has included in the `metadata.yaml` file, along with their various dependencies. + +This built charm is what will then be used by the SO to communicate with the +VNF. + +# Configuration + +The pingpong charm has several configuration properties that can be set via +the SO: + +- ssh-hostname +- ssh-username +- ssh-password +- ssh-private-key +- mode + +The ssh-* keys are included by the `sshproxy` layer, and enable the charm to +connect to the VNF image. + +The mode key must be one of two values: `ping` or `pong`. This informs the +charm as to which function it is serving. + +# Contact Information +For support, please send an email to the [OSM Tech] list. + + +[OSM Tech]: mailto:OSM_TECH@list.etsi.org +[Juju]: https://jujucharms.com/about +[configure]: https://jujucharms.com/docs/2.0/charms-config +[scaling]: https://jujucharms.com/docs/2.0/charms-scaling +[relations]: https://jujucharms.com/docs/2.0/charms-relations +[leadership]: https://jujucharms.com/docs/2.0/developer-leadership +[created your charm]: https://jujucharms.com/docs/2.0/developer-getting-started + + + + + +----- + + +# Integration + +After you've [created your charm], open `interfaces.yaml` and add +`layer:sshproxy` to the includes stanza, as shown below: +``` +includes: ['layer:basic', 'layer:sshproxy'] +``` + +## Reactive states + +This layer will set the following states: + +- `sshproxy.configured` This state is set when SSH credentials have been supplied to the charm. + + +## Example +In `reactive/mycharm.py`, you can add logic to execute commands over SSH. This +example is run via a `start` action, and starts a service running on a remote +host. +``` +... +import charms.sshproxy + + +@when('sshproxy.configured') +@when('actions.start') +def start(): + """ Execute's the command, via the start action` using the + configured SSH credentials + """ + sshproxy.ssh("service myservice start") + +``` + +## Actions +This layer includes a built-in `run` action useful for debugging or running arbitrary commands: + +``` +$ juju run-action mycharm/0 run command=hostname +Action queued with id: 014b72f3-bc02-4ecb-8d38-72bce03bbb63 + +$ juju show-action-output 014b72f3-bc02-4ecb-8d38-72bce03bbb63 +results: + output: juju-66a5f3-11 +status: completed +timing: + completed: 2016-10-27 19:53:49 +0000 UTC + enqueued: 2016-10-27 19:53:44 +0000 UTC + started: 2016-10-27 19:53:48 +0000 UTC + +``` +## Known Limitations and Issues + +### Security issues + +- Password and key-based authentications are supported, with the caveat that +both (password and private key) are stored plaintext within the Juju controller. + +# Configuration and Usage + +This layer adds the following configuration options: +- ssh-hostname +- ssh-username +- ssh-password +- ssh-private-key + +Once [configure] those values at any time. Once they are set, the `sshproxy.configured` state flag will be toggled: + +``` +juju deploy mycharm ssh-hostname=10.10.10.10 ssh-username=ubuntu ssh-password=yourpassword +``` +or +``` +juju deploy mycharm ssh-hostname=10.10.10.10 ssh-username=ubuntu ssh-private-key="cat `~/.ssh/id_rsa`" +``` + + +# Contact Information +Homepage: https://github.com/AdamIsrael/layer-sshproxy + +[Juju]: https://jujucharms.com/about +[configure]: https://jujucharms.com/docs/2.0/charms-config +[scaling]: https://jujucharms.com/docs/2.0/charms-scaling +[relations]: https://jujucharms.com/docs/2.0/charms-relations +[leadership]: https://jujucharms.com/docs/2.0/developer-leadership +[created your charm]: https://jujucharms.com/docs/2.0/developer-getting-started diff --git a/juju-charms/layers/pingpong/actions.yaml b/juju-charms/layers/pingpong/actions.yaml new file mode 100644 index 00000000..a5928f1f --- /dev/null +++ b/juju-charms/layers/pingpong/actions.yaml @@ -0,0 +1,32 @@ +set-server: + description: "Set the target IP address and port" + params: + server-ip: + description: "IP on which the target service is listening." + type: string + default: "" + server-port: + description: "Port on which the target service is listening." + type: integer + default: 5555 + required: + - server-ip +set-rate: + description: "Set the rate of packet generation." + params: + rate: + description: "Packet rate." + type: integer + default: 5 +get-stats: + description: "Get the stats." +get-state: + description: "Get the admin state of the target service." +get-rate: + description: "Get the rate set on the target service." +get-server: + description: "Get the target server and IP set" +start-traffic: + description: "Start the traffic generator or echo." +stop-traffic: + description: "Stop the traffic generator or echo." diff --git a/juju-charms/layers/pingpong/actions/get-rate b/juju-charms/layers/pingpong/actions/get-rate new file mode 100755 index 00000000..959b3e93 --- /dev/null +++ b/juju-charms/layers/pingpong/actions/get-rate @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.get-rate') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/pingpong/actions/get-server b/juju-charms/layers/pingpong/actions/get-server new file mode 100755 index 00000000..52e00894 --- /dev/null +++ b/juju-charms/layers/pingpong/actions/get-server @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.get-server') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/pingpong/actions/get-state b/juju-charms/layers/pingpong/actions/get-state new file mode 100755 index 00000000..446e8d71 --- /dev/null +++ b/juju-charms/layers/pingpong/actions/get-state @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.get-state') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/pingpong/actions/get-stats b/juju-charms/layers/pingpong/actions/get-stats new file mode 100755 index 00000000..086afc27 --- /dev/null +++ b/juju-charms/layers/pingpong/actions/get-stats @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.get-stats') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/pingpong/actions/set-rate b/juju-charms/layers/pingpong/actions/set-rate new file mode 100755 index 00000000..8fb723ef --- /dev/null +++ b/juju-charms/layers/pingpong/actions/set-rate @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.set-rate') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/pingpong/actions/set-server b/juju-charms/layers/pingpong/actions/set-server new file mode 100755 index 00000000..d1e908f5 --- /dev/null +++ b/juju-charms/layers/pingpong/actions/set-server @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.set-server') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/pingpong/actions/start-traffic b/juju-charms/layers/pingpong/actions/start-traffic new file mode 100755 index 00000000..562ac4c7 --- /dev/null +++ b/juju-charms/layers/pingpong/actions/start-traffic @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.start-traffic') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/pingpong/actions/stop-traffic b/juju-charms/layers/pingpong/actions/stop-traffic new file mode 100755 index 00000000..9352b331 --- /dev/null +++ b/juju-charms/layers/pingpong/actions/stop-traffic @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import sys +sys.path.append('lib') + +from charms.reactive import main +from charms.reactive import set_state +from charmhelpers.core.hookenv import action_fail + +""" +`set_state` only works here because it's flushed to disk inside the `main()` +loop. remove_state will need to be called inside the action method. +""" +set_state('actions.stop-traffic') + +try: + main() +except Exception as e: + action_fail(repr(e)) diff --git a/juju-charms/layers/pingpong/config.yaml b/juju-charms/layers/pingpong/config.yaml new file mode 100644 index 00000000..437524e4 --- /dev/null +++ b/juju-charms/layers/pingpong/config.yaml @@ -0,0 +1,5 @@ +options: + mode: + type: string + default: + description: "The service type: [ping, pong]" diff --git a/juju-charms/layers/pingpong/icon.svg b/juju-charms/layers/pingpong/icon.svg new file mode 100644 index 00000000..e092eef7 --- /dev/null +++ b/juju-charms/layers/pingpong/icon.svg @@ -0,0 +1,279 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + diff --git a/juju-charms/layers/pingpong/layer.yaml b/juju-charms/layers/pingpong/layer.yaml new file mode 100644 index 00000000..833eea68 --- /dev/null +++ b/juju-charms/layers/pingpong/layer.yaml @@ -0,0 +1,4 @@ +includes: + - layer:basic + - layer:vnfproxy +repo: https://osm.etsi.org/gerrit/osm/juju-charms diff --git a/juju-charms/layers/pingpong/metadata.yaml b/juju-charms/layers/pingpong/metadata.yaml new file mode 100644 index 00000000..1840743e --- /dev/null +++ b/juju-charms/layers/pingpong/metadata.yaml @@ -0,0 +1,13 @@ +name: pingpong +summary: +maintainer: Adam Israel +description: | + +tags: + # Replace "misc" with one or more whitelisted tags from this list: + # https://jujucharms.com/docs/stable/authors-charm-metadata + - misc +subordinate: false +series: + - trusty + - xenial diff --git a/juju-charms/layers/pingpong/reactive/pingpong.py b/juju-charms/layers/pingpong/reactive/pingpong.py new file mode 100644 index 00000000..b5a5db96 --- /dev/null +++ b/juju-charms/layers/pingpong/reactive/pingpong.py @@ -0,0 +1,272 @@ +from charmhelpers.core.hookenv import ( + action_get, + action_fail, + action_set, + config, + status_set, +) + +from charms.reactive import ( + remove_state as remove_flag, + set_state as set_flag, + when, +) +import charms.sshproxy +from subprocess import ( + Popen, + CalledProcessError, + PIPE, +) + + +cfg = config() + + +@when('config.changed') +def config_changed(): + if all(k in cfg for k in ['mode']): + if cfg['mode'] in ['ping', 'pong']: + set_flag('pingpong.configured') + status_set('active', 'ready!') + return + status_set('blocked', 'Waiting for configuration') + + +def is_ping(): + if cfg['mode'] == 'ping': + return True + return False + + +def is_pong(): + return not is_ping() + + +def get_port(): + port = 18888 + if is_pong(): + port = 18889 + return port + + +@when('pingpong.configured') +@when('actions.start') +def start(): + try: + # Bring up the eth1 interface. + # The selinux label on the file needs to be set correctly + cmd = "sudo timeout 5 /sbin/restorecon -v /etc/sysconfig/network-scripts/ifcfg-eth1" + result, err = charms.sshproxy._run(cmd) + except Exception as e: + err = "{}".format(e) + action_fail('command failed: {}, errors: {}'.format(err, e.output)) + remove_flag('actions.start') + return + + try: + cmd = "sudo timeout 30 /sbin/ifup eth1" + result, err = charms.sshproxy._run(cmd) + except Exception as e: + action_fail('command failed: {}, errors: {}'.format(e, e.output)) + remove_flag('actions.start') + return + + try: + cmd = "sudo timeout 30 /usr/bin/systemctl start {}". \ + format(cfg['mode']) + result, err = charms.sshproxy._run(cmd) + except Exception as e: + action_fail('command failed: {}, errors: {}'.format(e, e.output)) + else: + action_set({'stdout': result, + 'errors': err}) + finally: + remove_flag('actions.start') + + +@when('pingpong.configured') +@when('actions.stop') +def stop(): + try: + # Enter the command to stop your service(s) + cmd = "sudo timeout 30 /usr/bin/systemctl stop {}".format(cfg['mode']) + result, err = charms.sshproxy._run(cmd) + except Exception as e: + action_fail('command failed: {}, errors: {}'.format(e, e.output)) + else: + action_set({'stdout': result, + 'errors': err}) + finally: + remove_flag('actions.stop') + + +@when('pingpong.configured') +@when('actions.restart') +def restart(): + try: + # Enter the command to restart your service(s) + cmd = "sudo timeout 30 /usr/bin/systemctl restart {}".format(cfg['mode']) + result, err = charms.sshproxy._run(cmd) + except Exception as e: + action_fail('command failed: {}, errors: {}'.format(e, e.output)) + else: + action_set({'stdout': result, + 'errors': err}) + finally: + remove_flag('actions.restart') + + +@when('pingpong.configured') +@when('actions.set-server') +def set_server(): + try: + # Get the target service info + target_ip = action_get('server-ip') + target_port = action_get('server-port') + + data = '{{"ip" : "{}", "port" : {} }}'. \ + format(target_ip, target_port) + + cmd = format_curl( + 'POST', + '/server', + data, + ) + + result, err = charms.sshproxy._run(cmd) + except Exception as e: + action_fail('command failed: {}, errors: {}'.format(e, e.output)) + else: + action_set({'stdout': result, + 'errors': err}) + finally: + remove_flag('actions.set-server') + + +@when('pingpong.configured') +@when('actions.set-rate') +def set_rate(): + try: + if is_ping(): + rate = action_get('rate') + cmd = format_curl('POST', '/rate', '{{"rate" : {}}}'.format(rate)) + + result, err = charms.sshproxy._run(cmd) + except Exception as e: + err = "{}".format(e) + action_fail('command failed: {}, errors: {}'.format(err, e.output)) + else: + action_set({'stdout': result, + 'errors': err}) + finally: + remove_flag('actions.set-rate') + + +@when('pingpong.configured') +@when('actions.get-rate') +def get_rate(): + try: + if is_ping(): + cmd = format_curl('GET', '/rate') + + result, err = charms.sshproxy._run(cmd) + except Exception as e: + action_fail('command failed: {}, errors: {}'.format(e, e.output)) + else: + action_set({'stdout': result, + 'errors': err}) + finally: + remove_flag('actions.get-rate') + + +@when('pingpong.configured') +@when('actions.get-state') +def get_state(): + try: + cmd = format_curl('GET', '/state') + + result, err = charms.sshproxy._run(cmd) + except Exception as e: + action_fail('command failed: {}, errors: {}'.format(e, e.output)) + else: + action_set({'stdout': result, + 'errors': err}) + finally: + remove_flag('actions.get-state') + + +@when('pingpong.configured') +@when('actions.get-stats') +def get_stats(): + try: + cmd = format_curl('GET', '/stats') + + result, err = charms.sshproxy._run(cmd) + except Exception as e: + action_fail('command failed: {}, errors: {}'.format(e, e.output)) + else: + action_set({'stdout': result, + 'errors': err}) + finally: + remove_flag('actions.get-stats') + + +@when('pingpong.configured') +@when('actions.start-traffic') +def start_traffic(): + try: + cmd = format_curl('POST', '/adminstatus/state', '{"enable" : true}') + + result, err = charms.sshproxy._run(cmd) + except Exception as e: + action_fail('command failed: {}, errors: {}'.format(e, e.output)) + else: + action_set({'stdout': result, + 'errors': err}) + finally: + remove_flag('actions.start-traffic') + + +@when('pingpong.configured') +@when('actions.stop-traffic') +def stop_traffic(): + try: + cmd = format_curl('POST', '/adminstatus/state', '{"enable" : false}') + + result, err = charms.sshproxy._run(cmd) + except Exception as e: + action_fail('command failed: {}, errors: {}'.format(e, e.output)) + else: + action_set({'stdout': result, + 'errors': err}) + finally: + remove_flag('actions.stop-traffic') + + +def format_curl(method, path, data=None): + """ A utility function to build the curl command line. """ + + # method must be GET or POST + if method not in ['GET', 'POST']: + # Throw exception + return None + + # Get our service info + host = '127.0.0.1' + port = get_port() + mode = cfg['mode'] + + cmd = ['curl', + # '-D', '/dev/stdout', + '-H', 'Accept: application/vnd.yang.data+xml', + '-H', 'Content-Type: application/vnd.yang.data+json', + '-X', method] + + if method == "POST" and data: + cmd.append('-d') + cmd.append('{}'.format(data)) + + cmd.append( + 'http://{}:{}/api/v1/{}{}'.format(host, port, mode, path) + ) + return cmd diff --git a/juju-charms/layers/pingpong/tests/00-setup b/juju-charms/layers/pingpong/tests/00-setup new file mode 100755 index 00000000..f0616a56 --- /dev/null +++ b/juju-charms/layers/pingpong/tests/00-setup @@ -0,0 +1,5 @@ +#!/bin/bash + +sudo add-apt-repository ppa:juju/stable -y +sudo apt-get update +sudo apt-get install amulet python-requests -y diff --git a/juju-charms/layers/pingpong/tests/10-deploy b/juju-charms/layers/pingpong/tests/10-deploy new file mode 100755 index 00000000..d1d4719d --- /dev/null +++ b/juju-charms/layers/pingpong/tests/10-deploy @@ -0,0 +1,35 @@ +#!/usr/bin/python3 + +import amulet +import requests +import unittest + + +class TestCharm(unittest.TestCase): + def setUp(self): + self.d = amulet.Deployment() + + self.d.add('pingpong') + self.d.expose('pingpong') + + self.d.setup(timeout=900) + self.d.sentry.wait() + + self.unit = self.d.sentry['pingpong'][0] + + def test_service(self): + # test we can access over http + page = requests.get('http://{}'.format(self.unit.info['public-address'])) + self.assertEqual(page.status_code, 200) + # Now you can use self.d.sentry[SERVICE][UNIT] to address each of the units and perform + # more in-depth steps. Each self.d.sentry[SERVICE][UNIT] has the following methods: + # - .info - An array of the information of that unit from Juju + # - .file(PATH) - Get the details of a file on that unit + # - .file_contents(PATH) - Get plain text output of PATH file from that unit + # - .directory(PATH) - Get details of directory + # - .directory_contents(PATH) - List files and folders in PATH on that unit + # - .relation(relation, service:rel) - Get relation data from return service + + +if __name__ == '__main__': + unittest.main() diff --git a/juju-charms/layers/vyos-proxy/Makefile b/juju-charms/layers/vyos-proxy/Makefile new file mode 100644 index 00000000..a1ad3a5c --- /dev/null +++ b/juju-charms/layers/vyos-proxy/Makefile @@ -0,0 +1,24 @@ +#!/usr/bin/make + +all: lint unit_test + + +.PHONY: clean +clean: + @rm -rf .tox + +.PHONY: apt_prereqs +apt_prereqs: + @# Need tox, but don't install the apt version unless we have to (don't want to conflict with pip) + @which tox >/dev/null || (sudo apt-get install -y python-pip && sudo pip install tox) + +.PHONY: lint +lint: apt_prereqs + @tox --notest + @PATH=.tox/py34/bin:.tox/py35/bin flake8 $(wildcard hooks reactive lib unit_tests tests) + @charm proof + +.PHONY: unit_test +unit_test: apt_prereqs + @echo Starting tests... + tox diff --git a/juju-charms/layers/vyos-proxy/README.md b/juju-charms/layers/vyos-proxy/README.md new file mode 100644 index 00000000..0337c83b --- /dev/null +++ b/juju-charms/layers/vyos-proxy/README.md @@ -0,0 +1,221 @@ +# Overview + +This is the base layer for all charms [built using layers][building]. It +provides all of the standard Juju hooks and runs the +[charms.reactive.main][charms.reactive] loop for them. It also bootstraps the +[charm-helpers][] and [charms.reactive][] libraries and all of their +dependencies for use by the charm. + +# Usage + +To create a charm layer using this base layer, you need only include it in +a `layer.yaml` file: + +```yaml +includes: ['layer:basic'] +``` + +This will fetch this layer from [interfaces.juju.solutions][] and incorporate +it into your charm layer. You can then add handlers under the `reactive/` +directory. Note that **any** file under `reactive/` will be expected to +contain handlers, whether as Python decorated functions or [executables][non-python] +using the [external handler protocol][]. + +### Charm Dependencies + +Each layer can include a `wheelhouse.txt` file with Python requirement lines. +For example, this layer's `wheelhouse.txt` includes: + +``` +pip>=7.0.0,<8.0.0 +charmhelpers>=0.4.0,<1.0.0 +charms.reactive>=0.1.0,<2.0.0 +``` + +All of these dependencies from each layer will be fetched (and updated) at build +time and will be automatically installed by this base layer before any reactive +handlers are run. + +Note that the `wheelhouse.txt` file is intended for **charm** dependencies only. +That is, for libraries that the charm code itself needs to do its job of deploying +and configuring the payload. If the payload itself has Python dependencies, those +should be handled separately, by the charm. + +See [PyPI][pypi charms.X] for packages under the `charms.` namespace which might +be useful for your charm. + +### Layer Namespace + +Each layer has a reserved section in the `charms.layer.` Python package namespace, +which it can populate by including a `lib/charms/layer/.py` file or +by placing files under `lib/charms/layer//`. (If the layer name +includes hyphens, replace them with underscores.) These can be helpers that the +layer uses internally, or it can expose classes or functions to be used by other +layers to interact with that layer. + +For example, a layer named `foo` could include a `lib/charms/layer/foo.py` file +with some helper functions that other layers could access using: + +```python +from charms.layer.foo import my_helper +``` + +### Layer Options + +Any layer can define options in its `layer.yaml`. Those options can then be set +by other layers to change the behavior of your layer. The options are defined +using [jsonschema][], which is the same way that [action paramters][] are defined. + +For example, the `foo` layer could include the following option definitons: + +```yaml +includes: ['layer:basic'] +defines: # define some options for this layer (the layer "foo") + enable-bar: # define an "enable-bar" option for this layer + description: If true, enable support for "bar". + type: boolean + default: false +``` + +A layer using `foo` could then set it: + +```yaml +includes: ['layer:foo'] +options: + foo: # setting options for the "foo" layer + enable-bar: true # set the "enable-bar" option to true +``` + +The `foo` layer can then use the `charms.layer.options` helper to load the values +for the options that it defined. For example: + +```python +from charms import layer + +@when('state') +def do_thing(): + layer_opts = layer.options('foo') # load all of the options for the "foo" layer + if layer_opts['enable-bar']: # check the value of the "enable-bar" option + hookenv.log("Bar is enabled") +``` + +You can also access layer options in other handlers, such as Bash, using +the command-line interface: + +```bash +. charms.reactive.sh + +@when 'state' +function do_thing() { + if layer_option foo enable-bar; then + juju-log "Bar is enabled" + juju-log "bar-value is: $(layer_option foo bar-value)" + fi +} + +reactive_handler_main +``` + +Note that options of type `boolean` will set the exit code, while other types +will be printed out. + +# Hooks + +This layer provides hooks that other layers can react to using the decorators +of the [charms.reactive][] library: + + * `config-changed` + * `install` + * `leader-elected` + * `leader-settings-changed` + * `start` + * `stop` + * `upgrade-charm` + * `update-status` + +Other hooks are not implemented at this time. A new layer can implement storage +or relation hooks in their own layer by putting them in the `hooks` directory. + +**Note:** Because `update-status` is invoked every 5 minutes, you should take +care to ensure that your reactive handlers only invoke expensive operations +when absolutely necessary. It is recommended that you use helpers like +[`@only_once`][], [`@when_file_changed`][], and [`data_changed`][] to ensure +that handlers run only when necessary. + +# Layer Configuration + +This layer supports the following options, which can be set in `layer.yaml`: + + * **packages** A list of system packages to be installed before the reactive + handlers are invoked. + + * **use_venv** If set to true, the charm dependencies from the various + layers' `wheelhouse.txt` files will be installed in a Python virtualenv + located at `$CHARM_DIR/../.venv`. This keeps charm dependencies from + conflicting with payload dependencies, but you must take care to preserve + the environment and interpreter if using `execl` or `subprocess`. + + * **include_system_packages** If set to true and using a venv, include + the `--system-site-packages` options to make system Python libraries + visible within the venv. + +An example `layer.yaml` using these options might be: + +```yaml +includes: ['layer:basic'] +options: + basic: + packages: ['git'] + use_venv: true + include_system_packages: true +``` + + +# Reactive States + +This layer will set the following states: + + * **`config.changed`** Any config option has changed from its previous value. + This state is cleared automatically at the end of each hook invocation. + + * **`config.changed.