X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=installers%2Ffull_install_osm.sh;h=e759c56605f7e1b77ed63d8a19e137b310d70ee2;hb=c26d90bf8c425638f43f639b220ec09770a52e0c;hp=9ac0d8271c37eba8c4ade799525f75be3ba9c933;hpb=e32a5550944e00941b4caa4bdd22a57dd56263e4;p=osm%2Fdevops.git

diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh
index 9ac0d827..e759c566 100755
--- a/installers/full_install_osm.sh
+++ b/installers/full_install_osm.sh
@@ -26,11 +26,13 @@ function usage(){
     echo -e "                     -b v2.0            (v2.0 branch)"
     echo -e "                     -b tags/v1.1.0     (a specific tag)"
     echo -e "                     ..."
-    echo -e "     -s <stack name> user defined stack name, default is osm"
+    echo -e "     -c <orchestrator> deploy osm services using container <orchestrator>. Valid values are <k8s> or <swarm>.  If -c is not used then osm will be deployed using default orchestrator. When used with --uninstall, osm services deployed by the orchestrator will be uninstalled"
+    echo -e "     -s <stack name> or <namespace>  user defined stack name when installed using swarm or namespace when installed using k8s, default is osm"
     echo -e "     -H <VCA host>   use specific juju host controller IP"
     echo -e "     -S <VCA secret> use VCA/juju secret key"
     echo -e "     -P <VCA pubkey> use VCA/juju public key file"
     echo -e "     -C <VCA cacert> use VCA/juju CA certificate file"
+    echo -e "     -A <VCA apiproxy> use VCA/juju API proxy"
     echo -e "     --vimemu:       additionally deploy the VIM emulator as a docker container"
     echo -e "     --elk_stack:    additionally deploy an ELK docker stack for event logging"
     echo -e "     --pm_stack:     additionally deploy a Prometheus+Grafana stack for performance monitoring (PM)"
@@ -51,6 +53,7 @@ function usage(){
     echo -e "     --soui:         install classic build of OSM (Rel THREE v3.1, based on LXD containers, with SO and UI)"
     echo -e "     --lxdimages:    (only for Rel THREE with --soui) download lxd images from OSM repository instead of creating them from scratch"
     echo -e "     --pullimages:   pull/run osm images from docker.io/opensourcemano"
+    echo -e "     --k8s_monitor:  install the OSM kubernetes moitoring with prometheus and grafana"
     echo -e "     -l <lxd_repo>:  (only for Rel THREE with --soui) use specified repository url for lxd images"
     echo -e "     -p <path>:      (only for Rel THREE with --soui) use specified repository path for lxd images"
 #    echo -e "     --reconfigure:  reconfigure the modules (DO NOT change NAT rules)"
@@ -112,11 +115,17 @@ function generate_secret() {
 }
 
 function remove_volumes() {
-    stack=$1
-    volumes="mongo_db mon_db osm_packages ro_db"
-    for volume in $volumes; do
-        sg docker -c "docker volume rm ${stack}_${volume}"
-    done
+    if [ -n "$KUBERNETES" ]; then
+        k8_volume=$1
+        echo "Removing ${k8_volume}"
+        $WORKDIR_SUDO rm -rf ${k8_volume}
+    else
+        stack=$1
+        volumes="mongo_db mon_db osm_packages ro_db"
+        for volume in $volumes; do
+            sg docker -c "docker volume rm ${stack}_${volume}"
+        done
+    fi
 }
 
 function remove_network() {
@@ -124,6 +133,19 @@ function remove_network() {
     sg docker -c "docker network rm net${stack}"
 }
 
+function remove_iptables() {
+    stack=$1
+    if [ -z "$OSM_VCA_HOST" ]; then
+        OSM_VCA_HOST=`sg lxd -c "juju show-controller ${stack}"|grep api-endpoints|awk -F\' '{print $2}'|awk -F\: '{print $1}'`
+        [ -z "$OSM_VCA_HOST" ] && FATAL "Cannot obtain juju controller IP address"
+    fi
+
+    if sudo iptables -t nat -C PREROUTING -p tcp -m tcp --dport 17070 -j DNAT --to-destination $OSM_VCA_HOST; then
+        sudo iptables -t nat -D PREROUTING -p tcp -m tcp --dport 17070 -j DNAT --to-destination $OSM_VCA_HOST
+        sudo netfilter-persistent save
+    fi
+}
+
 function remove_stack() {
     stack=$1
     if sg docker -c "docker stack ps ${stack}" ; then
@@ -148,6 +170,11 @@ function remove_stack() {
     fi
 }
 
+#removes osm deployments and services
+function remove_k8s_namespace() {
+    kubectl delete ns $1
+}
+
 #Uninstall lightweight OSM: remove dockers
 function uninstall_lightweight() {
     if [ -n "$INSTALL_ONLY" ]; then
@@ -156,16 +183,19 @@ function uninstall_lightweight() {
             remove_stack osm_elk
             $WORKDIR_SUDO rm -rf $OSM_DOCKER_WORK_DIR/osm_elk
         fi
-        if [ -n "$INSTALL_PERFMON" ]; then
-            echo -e "\nUninstalling OSM Performance Monitoring stack"
-            remove_stack osm_metrics
-            $WORKDIR_SUDO rm -rf $OSM_DOCKER_WORK_DIR/osm_metrics
-        fi
     else
         echo -e "\nUninstalling OSM"
-        remove_stack $OSM_STACK_NAME
-        remove_stack osm_elk
-        remove_stack osm_metrics
+        if [ -n "$KUBERNETES" ]; then
+            if [ -n "$K8S_MONITOR" ]; then
+                # uninstall OSM MONITORING
+                uninstall_k8s_monitoring
+            fi
+            remove_k8s_namespace $OSM_STACK_NAME
+        else
+
+            remove_stack $OSM_STACK_NAME
+            remove_stack osm_elk
+        fi
         echo "Now osm docker images and volumes will be deleted"
         newgrp docker << EONG
         docker image rm ${DOCKER_USER}/ro:${OSM_DOCKER_TAG}
@@ -177,8 +207,15 @@ function uninstall_lightweight() {
         docker image rm ${DOCKER_USER}/pol:${OSM_DOCKER_TAG}
         docker image rm ${DOCKER_USER}/osmclient:${OSM_DOCKER_TAG}
 EONG
-        remove_volumes $OSM_STACK_NAME
-        remove_network $OSM_STACK_NAME
+
+        if [ -n "$KUBERNETES" ]; then
+            OSM_NAMESPACE_VOL="${OSM_HOST_VOL}/${OSM_STACK_NAME}"
+            remove_volumes $OSM_NAMESPACE_VOL
+        else
+            remove_volumes $OSM_STACK_NAME
+            remove_network $OSM_STACK_NAME
+        fi
+        remove_iptables $OSM_STACK_NAME
         echo "Removing $OSM_DOCKER_WORK_DIR"
         $WORKDIR_SUDO rm -rf $OSM_DOCKER_WORK_DIR
         sg lxd -c "juju destroy-controller --destroy-all-models --yes $OSM_STACK_NAME"
@@ -541,12 +578,12 @@ function install_osmclient(){
     CLIENT_REPOSITORY_BASE=${REPOSITORY_BASE#"-u "}
     key_location=$CLIENT_REPOSITORY_BASE/$CLIENT_RELEASE/$CLIENT_REPOSITORY_KEY
     curl $key_location | sudo apt-key add -
-    sudo add-apt-repository -y "deb [arch=amd64] $CLIENT_REPOSITORY_BASE/$CLIENT_RELEASE $CLIENT_REPOSITORY osmclient"
+    sudo add-apt-repository -y "deb [arch=amd64] $CLIENT_REPOSITORY_BASE/$CLIENT_RELEASE $CLIENT_REPOSITORY osmclient IM"
     sudo apt-get update
-    sudo apt-get install -y python-pip
-    sudo -H LC_ALL=C pip install -U pip
-    sudo -H LC_ALL=C python -m pip install python-magic
-    sudo apt-get install -y python-osmclient
+    sudo apt-get install -y python3-pip
+    sudo -H LC_ALL=C python3 -m pip install -U pip
+    sudo -H LC_ALL=C python3 -m pip install -U python-magic pyangbind
+    sudo apt-get install -y python3-osm-im python3-osmclient
     #sed 's,OSM_SOL005=[^$]*,OSM_SOL005=True,' -i ${HOME}/.bashrc
     #echo 'export OSM_HOSTNAME=localhost' >> ${HOME}/.bashrc
     #echo 'export OSM_SOL005=True' >> ${HOME}/.bashrc
@@ -565,6 +602,42 @@ function install_osmclient(){
     return 0
 }
 
+function install_prometheus_nodeexporter(){
+    if (systemctl -q is-active node_exporter)
+        then
+            echo "Node Exporter is already running."
+        else
+            echo "Node Exporter is not active, installing..."
+            if getent passwd node_exporter > /dev/null 2>&1; then
+                echo "node_exporter user exists"
+            else
+                echo "Creating user node_exporter"
+                sudo useradd --no-create-home --shell /bin/false node_exporter
+            fi
+            sudo wget -q https://github.com/prometheus/node_exporter/releases/download/v$PROMETHEUS_NODE_EXPORTER_TAG/node_exporter-$PROMETHEUS_NODE_EXPORTER_TAG.linux-amd64.tar.gz  -P /tmp/
+            sudo tar -C /tmp -xf /tmp/node_exporter-$PROMETHEUS_NODE_EXPORTER_TAG.linux-amd64.tar.gz
+            sudo cp /tmp/node_exporter-$PROMETHEUS_NODE_EXPORTER_TAG.linux-amd64/node_exporter /usr/local/bin
+            sudo chown node_exporter:node_exporter /usr/local/bin/node_exporter
+            sudo rm -rf node_exporter-$PROMETHEUS_NODE_EXPORTER_TAG.linux-amd64*
+            sudo cp ${OSM_DEVOPS}/installers/docker/files/node_exporter.service /etc/systemd/system/node_exporter.service
+            sudo systemctl daemon-reload
+            sudo systemctl restart node_exporter
+            sudo systemctl enable node_exporter
+            echo "Node Exporter has been activated in this host."
+    fi
+    return 0
+}
+
+function uninstall_prometheus_nodeexporter(){
+    sudo systemctl stop node_exporter
+    sudo systemctl disable node_exporter
+    sudo rm /etc/systemd/system/node_exporter.service
+    sudo systemctl daemon-reload
+    sudo userdel node_exporter
+    sudo rm /usr/local/bin/node_exporter
+    return 0
+}
+
 function install_from_lxdimages(){
     LXD_RELEASE=${RELEASE#"-R "}
     if [ -n "$LXD_REPOSITORY_PATH" ]; then
@@ -630,8 +703,9 @@ function install_docker_compose() {
 
 function install_juju() {
     echo "Installing juju"
-    sudo snap install juju --classic --channel=2.5/stable
+    sudo snap install juju --classic
     [ -z "$INSTALL_NOLXD" ] && sudo dpkg-reconfigure -p medium lxd
+    [[ ":$PATH": != *":/snap/bin:"* ]] && PATH="/snap/bin:${PATH}"
     echo "Finished installation of juju"
     return 0
 }
@@ -644,6 +718,17 @@ function juju_createcontroller() {
     [ $(juju controllers | awk "/^${OSM_STACK_NAME}[\*| ]/{print $1}"|wc -l) -eq 1 ] || FATAL "Juju installation failed"
 }
 
+function juju_createproxy() {
+    echo -e "\nChecking required packages: iptables-persistent"
+    dpkg -l iptables-persistent &>/dev/null || ! echo -e "    Not installed.\nInstalling iptables-persistent requires root privileges" || \
+    sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install iptables-persistent
+
+    if ! sudo iptables -t nat -C PREROUTING -p tcp -m tcp --dport 17070 -j DNAT --to-destination $OSM_VCA_HOST; then
+        sudo iptables -t nat -A PREROUTING -p tcp -m tcp --dport 17070 -j DNAT --to-destination $OSM_VCA_HOST
+        sudo netfilter-persistent save
+    fi
+}
+
 function generate_docker_images() {
     echo "Pulling and generating docker images"
     _build_from=$COMMIT_ID
@@ -669,6 +754,10 @@ function generate_docker_images() {
         sg docker -c "docker pull prom/prometheus:${PROMETHEUS_TAG}" || FATAL "cannot get prometheus docker image"
     fi
 
+    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q GRAFANA ; then
+        sg docker -c "docker pull grafana/grafana:${GRAFANA_TAG}" || FATAL "cannot get grafana docker image"
+    fi    
+
     if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q NBI || echo $TO_REBUILD | grep -q KEYSTONE-DB ; then
         sg docker -c "docker pull mariadb:${KEYSTONEDB_TAG}" || FATAL "cannot get keystone-db docker image"
     fi
@@ -708,7 +797,7 @@ function generate_docker_images() {
     elif [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q RO ; then
         git -C ${LWTEMPDIR} clone https://osm.etsi.org/gerrit/osm/RO
         git -C ${LWTEMPDIR}/RO checkout ${COMMIT_ID}
-        sg docker -c "docker build ${LWTEMPDIR}/RO -f ${LWTEMPDIR}/RO/docker/Dockerfile-local -t ${DOCKER_USER}/ro --no-cache" || FATAL "cannot build RO docker image"
+        sg docker -c "docker build ${LWTEMPDIR}/RO -f ${LWTEMPDIR}/RO/Dockerfile-local -t ${DOCKER_USER}/ro --no-cache" || FATAL "cannot build RO docker image"
     fi
 
     if [ -n "$PULL_IMAGES" ]; then
@@ -732,6 +821,11 @@ function generate_docker_images() {
     elif [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q LW-osmclient; then
         sg docker -c "docker build -t ${DOCKER_USER}/osmclient ${BUILD_ARGS[@]} -f $OSM_DEVOPS/docker/osmclient ."
     fi
+
+    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q PROMETHEUS ; then
+        sg docker -c "docker pull google/cadvisor:${PROMETHEUS_CADVISOR_TAG}" || FATAL "cannot get prometheus cadvisor docker image"
+    fi    
+
     echo "Finished generation of docker images"
 }
 
@@ -747,15 +841,34 @@ function cmp_overwrite() {
     fi
 }
 
-function generate_config_log_folders() {
-    echo "Generating config and log folders"
-    $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/docker-compose.yaml $OSM_DOCKER_WORK_DIR/docker-compose.yaml
-    $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/prometheus.yml $OSM_DOCKER_WORK_DIR/prometheus.yml
-    echo "Finished generation of config and log folders"
-}
-
 function generate_docker_env_files() {
+    echo "Doing a backup of existing env files"
+    $WORKDIR_SUDO cp $OSM_DOCKER_WORK_DIR/keystone-db.env{,~}
+    $WORKDIR_SUDO cp $OSM_DOCKER_WORK_DIR/keystone.env{,~}
+    $WORKDIR_SUDO cp $OSM_DOCKER_WORK_DIR/lcm.env{,~}
+    $WORKDIR_SUDO cp $OSM_DOCKER_WORK_DIR/lwui.env{,~}
+    $WORKDIR_SUDO cp $OSM_DOCKER_WORK_DIR/mon.env{,~}
+    $WORKDIR_SUDO cp $OSM_DOCKER_WORK_DIR/nbi.env{,~}
+    $WORKDIR_SUDO cp $OSM_DOCKER_WORK_DIR/pol.env{,~}
+    $WORKDIR_SUDO cp $OSM_DOCKER_WORK_DIR/ro-db.env{,~}
+    $WORKDIR_SUDO cp $OSM_DOCKER_WORK_DIR/ro.env{,~}
+
     echo "Generating docker env files"
+    if [ -n "$KUBERNETES" ]; then
+        #Kubernetes resources
+        $WORKDIR_SUDO cp -bR ${OSM_DEVOPS}/installers/docker/osm_pods $OSM_DOCKER_WORK_DIR
+    else
+        # Docker-compose
+        $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/docker-compose.yaml $OSM_DOCKER_WORK_DIR/docker-compose.yaml
+
+        # Prometheus
+        $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/files/prometheus.yml $OSM_DOCKER_WORK_DIR/prometheus.yml
+
+        # Grafana & Prometheus Exporter files
+        $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/files
+        $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/files/* $OSM_DOCKER_WORK_DIR/files/
+    fi
+
     # LCM
     if [ ! -f $OSM_DOCKER_WORK_DIR/lcm.env ]; then
         echo "OSMLCM_DATABASE_COMMONKEY=${OSM_DATABASE_COMMONKEY}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/lcm.env
@@ -779,11 +892,17 @@ function generate_docker_env_files() {
         $WORKDIR_SUDO sed -i "s|OSMLCM_VCA_PUBKEY.*|OSMLCM_VCA_PUBKEY=\"${OSM_VCA_PUBKEY}\"|g" $OSM_DOCKER_WORK_DIR/lcm.env
     fi
 
-    #if ! grep -Fq "OSMLCM_VCA_CACERT" $OSM_DOCKER_WORK_DIR/lcm.env; then
-    #    echo "OSMLCM_VCA_CACERT=\"${OSM_VCA_CACERT}\"" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/lcm.env
-    #else
-    #    $WORKDIR_SUDO sed -i "s|OSMLCM_VCA_CACERT.*|OSMLCM_VCA_CACERT=\"${OSM_VCA_CACERT}\"|g" $OSM_DOCKER_WORK_DIR/lcm.env
-    #fi
+    if ! grep -Fq "OSMLCM_VCA_CACERT" $OSM_DOCKER_WORK_DIR/lcm.env; then
+       echo "OSMLCM_VCA_CACERT=${OSM_VCA_CACERT}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/lcm.env
+    else
+       $WORKDIR_SUDO sed -i "s|OSMLCM_VCA_CACERT.*|OSMLCM_VCA_CACERT=${OSM_VCA_CACERT}|g" $OSM_DOCKER_WORK_DIR/lcm.env
+    fi
+
+    if ! grep -Fq "OSMLCM_VCA_APIPROXY" $OSM_DOCKER_WORK_DIR/lcm.env; then
+        echo "OSMLCM_VCA_APIPROXY=${OSM_VCA_APIPROXY}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/lcm.env
+    else
+        $WORKDIR_SUDO sed -i "s|OSMLCM_VCA_APIPROXY.*|OSMLCM_VCA_APIPROXY=${OSM_VCA_APIPROXY}|g" $OSM_DOCKER_WORK_DIR/lcm.env
+    fi
 
     # RO
     MYSQL_ROOT_PASSWORD=$(generate_secret)
@@ -836,6 +955,13 @@ function generate_docker_env_files() {
         $WORKDIR_SUDO sed -i "s|OSMMON_VCA_SECRET.*|OSMMON_VCA_SECRET=$OSM_VCA_SECRET|g" $OSM_DOCKER_WORK_DIR/mon.env
     fi
 
+    if ! grep -Fq "OSMMON_VCA_CACERT" $OSM_DOCKER_WORK_DIR/mon.env; then
+       echo "OSMMON_VCA_CACERT=${OSM_VCA_CACERT}" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/mon.env
+    else
+       $WORKDIR_SUDO sed -i "s|OSMMON_VCA_CACERT.*|OSMMON_VCA_CACERT=${OSM_VCA_CACERT}|g" $OSM_DOCKER_WORK_DIR/mon.env
+    fi
+
+
     # POL
     if [ ! -f $OSM_DOCKER_WORK_DIR/pol.env ]; then
         echo "OSMPOL_SQL_DATABASE_URI=mysql://root:${MYSQL_ROOT_PASSWORD}@mysql:3306/pol" | $WORKDIR_SUDO tee -a $OSM_DOCKER_WORK_DIR/pol.env
@@ -855,6 +981,75 @@ function generate_osmclient_script () {
     echo "osmclient sidecar container can be found at: $OSM_DOCKER_WORK_DIR/osm"
 }
 
+#installs kubernetes packages
+function install_kube() {
+    sudo apt-get update && sudo apt-get install -y apt-transport-https
+    curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+    sudo add-apt-repository "deb https://apt.kubernetes.io/ kubernetes-xenial main"
+    sudo apt-get update
+    echo "Installing Kubernetes Packages ..."
+    sudo apt-get install -y kubelet=1.15.0-00 kubeadm=1.15.0-00 kubectl=1.15.0-00
+}
+
+#initializes kubernetes control plane
+function init_kubeadm() {
+    sudo swapoff -a
+    sudo kubeadm init --config $1
+    sleep 5
+}
+
+function kube_config_dir() {
+    [ ! -d $K8S_MANIFEST_DIR ] && FATAL "Cannot Install Kubernetes"
+    mkdir -p $HOME/.kube
+    sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config
+    sudo chown $(id -u):$(id -g) $HOME/.kube/config
+}
+
+#deploys flannel as daemonsets
+function deploy_cni_provider() {
+    CNI_DIR="$(mktemp -d -q --tmpdir "flannel.XXXXXX")"
+    trap 'rm -rf "${CNI_DIR}"' EXIT
+    wget -q https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml -P $CNI_DIR
+    kubectl apply -f $CNI_DIR
+    [ $? -ne 0 ] && FATAL "Cannot Install Flannel"
+}
+
+#creates secrets from env files which will be used by containers
+function kube_secrets(){
+    kubectl create ns $OSM_STACK_NAME
+    kubectl create secret generic lcm-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/lcm.env
+    kubectl create secret generic mon-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/mon.env
+    kubectl create secret generic nbi-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/nbi.env
+    kubectl create secret generic ro-db-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/ro-db.env
+    kubectl create secret generic ro-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/ro.env
+    kubectl create secret generic keystone-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/keystone.env
+    kubectl create secret generic lwui-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/lwui.env
+    kubectl create secret generic pol-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/pol.env
+}
+
+#deploys osm pods and services
+function deploy_osm_services() {
+    K8S_MASTER=$(kubectl get nodes | awk '$3~/master/'| awk '{print $1}')
+    kubectl taint node $K8S_MASTER node-role.kubernetes.io/master:NoSchedule-
+    sleep 5
+    kubectl apply -n $OSM_STACK_NAME -f $OSM_K8S_WORK_DIR
+}
+
+function parse_yaml() {
+    osm_services="nbi lcm ro pol mon light-ui keystone"
+    TAG=$1
+    for osm in $osm_services; do
+        $WORKDIR_SUDO sed -i "s/opensourcemano\/$osm:.*/opensourcemano\/$osm:$TAG/g" $OSM_K8S_WORK_DIR/$osm.yaml
+    done
+}
+
+function namespace_vol() {
+    osm_services="nbi lcm ro pol mon kafka mongo mysql"
+    for osm in $osm_services; do
+          $WORKDIR_SUDO  sed -i "s#path: /var/lib/osm#path: $OSM_NAMESPACE_VOL#g" $OSM_K8S_WORK_DIR/$osm.yaml
+    done
+}
+
 function init_docker_swarm() {
     if [ "${DEFAULT_MTU}" != "1500" ]; then
       DOCKER_NETS=`sg docker -c "docker network list" | awk '{print $2}' | egrep -v "^ID$" | paste -d " " -s`
@@ -880,9 +1075,11 @@ function deploy_lightweight() {
     OSM_UI_PORT=80
     OSM_MON_PORT=8662
     OSM_PROM_PORT=9090
+    OSM_PROM_CADVISOR_PORT=8080
     OSM_PROM_HOSTPORT=9091
+    OSM_GRAFANA_PORT=3000
     [ -n "$INSTALL_ELK" ] && OSM_ELK_PORT=5601
-    [ -n "$INSTALL_PERFMON" ] && OSM_PM_PORT=3000
+    #[ -n "$INSTALL_PERFMON" ] && OSM_PM_PORT=3000
 
     if [ -n "$NO_HOST_PORTS" ]; then
         OSM_PORTS+=(OSM_NBI_PORTS=$OSM_NBI_PORT)
@@ -891,7 +1088,9 @@ function deploy_lightweight() {
         OSM_PORTS+=(OSM_UI_PORTS=$OSM_UI_PORT)
         OSM_PORTS+=(OSM_MON_PORTS=$OSM_MON_PORT)
         OSM_PORTS+=(OSM_PROM_PORTS=$OSM_PROM_PORT)
-        [ -n "$INSTALL_PERFMON" ] && OSM_PORTS+=(OSM_PM_PORTS=$OSM_PM_PORT)
+        OSM_PORTS+=(OSM_PROM_CADVISOR_PORTS=$OSM_PROM_CADVISOR_PORT)
+        OSM_PORTS+=(OSM_GRAFANA_PORTS=$OSM_GRAFANA_PORT)
+        #[ -n "$INSTALL_PERFMON" ] && OSM_PORTS+=(OSM_PM_PORTS=$OSM_PM_PORT)
         [ -n "$INSTALL_ELK" ] && OSM_PORTS+=(OSM_ELK_PORTS=$OSM_ELK_PORT)
     else
         OSM_PORTS+=(OSM_NBI_PORTS=$OSM_NBI_PORT:$OSM_NBI_PORT)
@@ -900,7 +1099,9 @@ function deploy_lightweight() {
         OSM_PORTS+=(OSM_UI_PORTS=$OSM_UI_PORT:$OSM_UI_PORT)
         OSM_PORTS+=(OSM_MON_PORTS=$OSM_MON_PORT:$OSM_MON_PORT)
         OSM_PORTS+=(OSM_PROM_PORTS=$OSM_PROM_HOSTPORT:$OSM_PROM_PORT)
-        [ -n "$INSTALL_PERFMON" ] && OSM_PORTS+=(OSM_PM_PORTS=$OSM_PM_PORT:$OSM_PM_PORT)
+        OSM_PORTS+=(OSM_PROM_CADVISOR_PORTS=$OSM_PROM_CADVISOR_PORT:$OSM_PROM_CADVISOR_PORT)
+        OSM_PORTS+=(OSM_GRAFANA_PORTS=$OSM_GRAFANA_PORT:$OSM_GRAFANA_PORT)
+        #[ -n "$INSTALL_PERFMON" ] && OSM_PORTS+=(OSM_PM_PORTS=$OSM_PM_PORT:$OSM_PM_PORT)
         [ -n "$INSTALL_ELK" ] && OSM_PORTS+=(OSM_ELK_PORTS=$OSM_ELK_PORT:$OSM_ELK_PORT)
     fi
     echo "export ${OSM_PORTS[@]}" | $WORKDIR_SUDO tee $OSM_DOCKER_WORK_DIR/osm_ports.sh
@@ -910,6 +1111,8 @@ function deploy_lightweight() {
     echo "export KAFKA_TAG=${KAFKA_TAG}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
     echo "export PROMETHEUS_TAG=${PROMETHEUS_TAG}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
     echo "export KEYSTONEDB_TAG=${KEYSTONEDB_TAG}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
+    echo "export PROMETHEUS_CADVISOR_TAG=${PROMETHEUS_CADVISOR_TAG}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
+    echo "export GRAFANA_TAG=${GRAFANA_TAG}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
 
     pushd $OSM_DOCKER_WORK_DIR
     sg docker -c ". ./osm_ports.sh; docker stack deploy -c $OSM_DOCKER_WORK_DIR/docker-compose.yaml $OSM_STACK_NAME"
@@ -968,29 +1171,31 @@ function deploy_elk() {
     return 0
 }
 
-function deploy_perfmon() {
-    echo "Pulling docker images for PM (Grafana)"
-    sg docker -c "docker pull grafana/grafana" || FATAL "cannot get grafana docker image"
-    echo "Finished pulling PM docker images"
-    $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/osm_metrics
-    $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/osm_metrics/*.yml $OSM_DOCKER_WORK_DIR/osm_metrics
-    $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/osm_metrics/*.json $OSM_DOCKER_WORK_DIR/osm_metrics
-    remove_stack osm_metrics
-    echo "Deploying PM stack (Grafana)"
-    sg docker -c "OSM_NETWORK=net${OSM_STACK_NAME} docker stack deploy -c $OSM_DOCKER_WORK_DIR/osm_metrics/docker-compose.yml osm_metrics"
-    echo "Finished deployment of PM stack"
-    return 0
-}
-
 function install_lightweight() {
     [ "${OSM_STACK_NAME}" == "osm" ] || OSM_DOCKER_WORK_DIR="$OSM_WORK_DIR/stack/$OSM_STACK_NAME"
+    [ -n "$KUBERNETES" ] && OSM_K8S_WORK_DIR="$OSM_DOCKER_WORK_DIR/osm_pods" && OSM_NAMESPACE_VOL="${OSM_HOST_VOL}/${OSM_STACK_NAME}"
     [ ! -d "$OSM_DOCKER_WORK_DIR" ] && $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR
+    [ -n "$KUBERNETES" ] && $WORKDIR_SUDO cp -b $OSM_DEVOPS/installers/docker/cluster-config.yaml $OSM_DOCKER_WORK_DIR/cluster-config.yaml
 
     track checkingroot
     [ "$USER" == "root" ] && FATAL "You are running the installer as root. The installer is prepared to be executed as a normal user with sudo privileges."
     track noroot
-    [ -z "$ASSUME_YES" ] && ! ask_user "The installation will configure LXD, install juju, install docker CE and init a docker swarm, as pre-requirements. Do you want to proceed (Y/n)? " y && echo "Cancelled!" && exit 1
+
+    if [ -n "$KUBERNETES" ]; then
+        [ -z "$ASSUME_YES" ] && ! ask_user "The installation will do the following
+        1. Install and configure LXD
+        2. Install juju
+        3. Install docker CE
+        4. Disable swap space
+        5. Install and initialize Kubernetes
+        as pre-requirements.
+        Do you want to proceed (Y/n)? " y && echo "Cancelled!" && exit 1
+
+    else
+        [ -z "$ASSUME_YES" ] && ! ask_user "The installation will configure LXD, install juju, install docker CE and init a docker swarm, as pre-requirements. Do you want to proceed (Y/n)? " y && echo "Cancelled!" && exit 1
+    fi
     track proceed
+
     echo "Installing lightweight build of OSM"
     LWTEMPDIR="$(mktemp -d -q --tmpdir "installosmlight.XXXXXX")"
     trap 'rm -rf "${LWTEMPDIR}"' EXIT
@@ -1014,15 +1219,17 @@ function install_lightweight() {
           || FATAL "failed to install $need_packages_lw"
     fi
     track prereqok
-    [ -z "$INSTALL_NOJUJU" ] && install_juju
 
+    [ -z "$INSTALL_NOJUJU" ] && install_juju
     track juju_install
+
     if [ -z "$OSM_VCA_HOST" ]; then
         juju_createcontroller
         OSM_VCA_HOST=`sg lxd -c "juju show-controller $OSM_STACK_NAME"|grep api-endpoints|awk -F\' '{print $2}'|awk -F\: '{print $1}'`
         [ -z "$OSM_VCA_HOST" ] && FATAL "Cannot obtain juju controller IP address"
     fi
     track juju_controller
+
     if [ -z "$OSM_VCA_SECRET" ]; then
         OSM_VCA_SECRET=$(parse_juju_password $OSM_STACK_NAME)
         [ -z "$OSM_VCA_SECRET" ] && FATAL "Cannot obtain juju secret"
@@ -1031,38 +1238,78 @@ function install_lightweight() {
         OSM_VCA_PUBKEY=$(cat $HOME/.local/share/juju/ssh/juju_id_rsa.pub)
         [ -z "$OSM_VCA_PUBKEY" ] && FATAL "Cannot obtain juju public key"
     fi
-    #if [ -z "$OSM_VCA_CACERT" ]; then
-	#OSM_VCA_CACERT=$(juju controllers --format json | jq -r '.controllers["osm"]["ca-cert"]' | grep -v "\-\-\-\-\-.*CERTIFICATE\-\-\-\-\-")
-    #    [ -z "$OSM_VCA_CACERT" ] && FATAL "Cannot obtain juju CA certificate"
-    #fi
+    if [ -z "$OSM_VCA_APIPROXY" ]; then
+        OSM_VCA_APIPROXY=$DEFAULT_IP
+        [ -z "$OSM_VCA_APIPROXY" ] && FATAL "Cannot obtain juju api proxy"
+    fi
+    juju_createproxy
+
+    if [ -z "$OSM_VCA_CACERT" ]; then
+	OSM_VCA_CACERT=$(juju controllers --format json | jq -r '.controllers["osm"]["ca-cert"]' | base64 | tr -d \\n)
+       [ -z "$OSM_VCA_CACERT" ] && FATAL "Cannot obtain juju CA certificate"
+    fi
     if [ -z "$OSM_DATABASE_COMMONKEY" ]; then
         OSM_DATABASE_COMMONKEY=$(generate_secret)
         [ -z "OSM_DATABASE_COMMONKEY" ] && FATAL "Cannot generate common db secret"
     fi
-
     track juju
+
     [ -n "$INSTALL_NODOCKER" ] || install_docker_ce
     track docker_ce
-    #install_docker_compose
-    [ -n "$INSTALL_NODOCKER" ] || init_docker_swarm
-    track docker_swarm
+
+    #Installs Kubernetes and deploys osm services
+    if [ -n "$KUBERNETES" ]; then
+        install_kube
+        track install_k8s
+        init_kubeadm $OSM_DOCKER_WORK_DIR/cluster-config.yaml
+        kube_config_dir
+        track init_k8s
+    else
+        #install_docker_compose
+        [ -n "$INSTALL_NODOCKER" ] || init_docker_swarm
+        track docker_swarm
+    fi
+
     [ -z "$DOCKER_NOBUILD" ] && generate_docker_images
     track docker_build
+
     generate_docker_env_files
-    generate_config_log_folders
-
-    # remove old stack
-    remove_stack $OSM_STACK_NAME
-    create_docker_network
-    deploy_lightweight
-    generate_osmclient_script
-    track docker_deploy
-    [ -n "$INSTALL_VIMEMU" ] && install_vimemu && track vimemu
-    [ -n "$INSTALL_ELK" ] && deploy_elk && track elk
-    [ -n "$INSTALL_PERFMON" ] && deploy_perfmon && track perfmon
+
+    if [ -n "$KUBERNETES" ]; then
+        if [ -n "$K8S_MONITOR" ]; then
+            # uninstall OSM MONITORING
+            uninstall_k8s_monitoring
+        fi
+        #remove old namespace
+        remove_k8s_namespace $OSM_STACK_NAME
+        deploy_cni_provider
+        kube_secrets
+        [ ! $OSM_DOCKER_TAG == "latest" ] && parse_yaml $OSM_DOCKER_TAG
+        namespace_vol
+        deploy_osm_services
+        track deploy_osm_services_k8s
+    else
+        # remove old stack
+        remove_stack $OSM_STACK_NAME
+        create_docker_network
+        deploy_lightweight
+        generate_osmclient_script
+        track docker_deploy
+        install_prometheus_nodeexporter
+        [ -n "$INSTALL_VIMEMU" ] && install_vimemu && track vimemu
+        [ -n "$INSTALL_ELK" ] && deploy_elk && track elk
+    fi
+
+    if [ -n "$KUBERNETES" ] && [ -n "$K8S_MONITOR" ]; then
+        # install OSM MONITORING
+        install_k8s_monitoring
+        track install_k8s_monitoring
+    fi
+
     [ -z "$INSTALL_NOHOSTCLIENT" ] && install_osmclient
     track osmclient
-    wget -q -O- https://osm-download.etsi.org/ftp/osm-6.0-six/sixME2.txt &> /dev/null
+
+    wget -q -O- https://osm-download.etsi.org/ftp/osm-6.0-six/README2.txt &> /dev/null
     track end
     return 0
 }
@@ -1098,6 +1345,18 @@ function install_vimemu() {
     echo "     osm vim-create --name emu-vim1 --user username --password password --auth_url http://${VIMEMU_HOSTNAME}:6001/v2.0 --tenant tenantName --account_type openstack"
 }
 
+function install_k8s_monitoring() {
+    # install OSM monitoring
+    chmod +x $WORKDIR_SUDO $OSM_DEVOPS/installers/k8s/*.sh
+    $WORKDIR_SUDO $OSM_DEVOPS/installers/k8s/install_osm_k8s_monitoring.sh
+}
+
+function uninstall_k8s_monitoring() {
+    # install OSM monitoring
+    chmod +x $WORKDIR_SUDO $OSM_DEVOPS/installers/k8s/*.sh
+    $WORKDIR_SUDO $OSM_DEVOPS/installers/k8s/uninstall_osm_k8s_monitoring.sh
+}
+
 function dump_vars(){
     echo "DEVELOP=$DEVELOP"
     echo "INSTALL_FROM_SOURCE=$INSTALL_FROM_SOURCE"
@@ -1114,7 +1373,7 @@ function dump_vars(){
     echo "INSTALL_LIGHTWEIGHT=$INSTALL_LIGHTWEIGHT"
     echo "INSTALL_ONLY=$INSTALL_ONLY"
     echo "INSTALL_ELK=$INSTALL_ELK"
-    echo "INSTALL_PERFMON=$INSTALL_PERFMON"
+    #echo "INSTALL_PERFMON=$INSTALL_PERFMON"
     echo "TO_REBUILD=$TO_REBUILD"
     echo "INSTALL_NOLXD=$INSTALL_NOLXD"
     echo "INSTALL_NODOCKER=$INSTALL_NODOCKER"
@@ -1136,6 +1395,7 @@ function dump_vars(){
     echo "DOCKER_USER=$DOCKER_USER"
     echo "OSM_STACK_NAME=$OSM_STACK_NAME"
     echo "PULL_IMAGES=$PULL_IMAGES"
+    echo "KUBERNETES=$KUBERNETES"
     echo "SHOWOPTS=$SHOWOPTS"
     echo "Install from specific refspec (-b): $COMMIT_ID"
 }
@@ -1174,11 +1434,13 @@ LXD_REPOSITORY_PATH=""
 INSTALL_LIGHTWEIGHT="y"
 INSTALL_ONLY=""
 INSTALL_ELK=""
-INSTALL_PERFMON=""
+#INSTALL_PERFMON=""
 TO_REBUILD=""
 INSTALL_NOLXD=""
 INSTALL_NODOCKER=""
 INSTALL_NOJUJU=""
+KUBERNETES=""
+K8S_MONITOR=""
 INSTALL_NOHOSTCLIENT=""
 NOCONFIGURE=""
 RELEASE_DAILY=""
@@ -1195,17 +1457,26 @@ REPOSITORY_BASE="https://osm-download.etsi.org/repository/osm/debian"
 WORKDIR_SUDO=sudo
 OSM_WORK_DIR="/etc/osm"
 OSM_DOCKER_WORK_DIR="/etc/osm/docker"
+OSM_K8S_WORK_DIR="${OSM_DOCKER_WORK_DIR}/osm_pods"
+OSM_HOST_VOL="/var/lib/osm"
+OSM_NAMESPACE_VOL="${OSM_HOST_VOL}/${OSM_STACK_NAME}"
 OSM_DOCKER_TAG=latest
 DOCKER_USER=opensourcemano
 PULL_IMAGES="y"
 KAFKA_TAG=2.11-1.0.2
 PROMETHEUS_TAG=v2.4.3
+GRAFANA_TAG=latest
+PROMETHEUS_NODE_EXPORTER_TAG=0.18.1
+PROMETHEUS_CADVISOR_TAG=latest
 KEYSTONEDB_TAG=10
 OSM_DATABASE_COMMONKEY=
 ELASTIC_VERSION=6.4.2
 ELASTIC_CURATOR_VERSION=5.5.4
+POD_NETWORK_CIDR=10.244.0.0/16
+K8S_MANIFEST_DIR="/etc/kubernetes/manifests"
+RE_CHECK='^[a-z0-9]([-a-z0-9]*[a-z0-9])?$'
 
-while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:w:t:U:P:" o; do
+while getopts ":hy-:b:r:c:k:u:R:l:p:D:o:m:H:S:s:w:t:U:P:A:" o; do
     case "${o}" in
         h)
             usage && exit 0
@@ -1218,6 +1489,12 @@ while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:w:t:U:P:" o; do
             REPOSITORY="${OPTARG}"
             REPO_ARGS+=(-r "$REPOSITORY")
             ;;
+        c)
+            [ "${OPTARG}" == "swarm" ] && continue
+            [ "${OPTARG}" == "k8s" ] && KUBERNETES="y" && continue
+            echo -e "Invalid argument for -i : ' $OPTARG'\n" >&2
+            usage && exit 1
+            ;;
         R)
             RELEASE="${OPTARG}"
             REPO_ARGS+=(-R "$RELEASE")
@@ -1243,7 +1520,7 @@ while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:w:t:U:P:" o; do
             OSM_DEVOPS="${OPTARG}"
             ;;
         s)
-            OSM_STACK_NAME="${OPTARG}"
+            OSM_STACK_NAME="${OPTARG}" && [ -n "$KUBERNETES" ] && [[ ! "${OPTARG}" =~ $RE_CHECK ]] && echo "Namespace $OPTARG is invalid. Regex used for validation is $RE_CHECK" && exit 0
             ;;
         H)
             OSM_VCA_HOST="${OPTARG}"
@@ -1254,6 +1531,9 @@ while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:w:t:U:P:" o; do
         P)
             OSM_VCA_PUBKEY=$(cat ${OPTARG})
             ;;
+        A)
+            OSM_VCA_APIPROXY="${OPTARG}"
+            ;;
         w)
             # when specifying workdir, do not use sudo for access
             WORKDIR_SUDO=
@@ -1266,7 +1546,7 @@ while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:w:t:U:P:" o; do
             INSTALL_ONLY="y"
             [ "${OPTARG}" == "vimemu" ] && INSTALL_VIMEMU="y" && continue
             [ "${OPTARG}" == "elk_stack" ] && INSTALL_ELK="y" && continue
-            [ "${OPTARG}" == "pm_stack" ] && INSTALL_PERFMON="y" && continue
+            #[ "${OPTARG}" == "pm_stack" ] && INSTALL_PERFMON="y" && continue
             ;;
         m)
             [ "${OPTARG}" == "LW-UI" ] && TO_REBUILD="$TO_REBUILD LW-UI" && continue
@@ -1279,6 +1559,7 @@ while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:w:t:U:P:" o; do
             [ "${OPTARG}" == "MONGO" ] && TO_REBUILD="$TO_REBUILD MONGO" && continue
             [ "${OPTARG}" == "PROMETHEUS" ] && TO_REBUILD="$TO_REBUILD PROMETHEUS" && continue
             [ "${OPTARG}" == "KEYSTONE-DB" ] && TO_REBUILD="$TO_REBUILD KEYSTONE-DB" && continue
+            [ "${OPTARG}" == "GRAFANA" ] && TO_REBUILD="$TO_REBUILD GRAFANA" && continue
             [ "${OPTARG}" == "NONE" ] && TO_REBUILD="$TO_REBUILD NONE" && continue
             ;;
         -)
@@ -1298,7 +1579,7 @@ while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:w:t:U:P:" o; do
             [ "${OPTARG}" == "soui" ] && INSTALL_LIGHTWEIGHT="" && RELEASE="-R ReleaseTHREE" && REPOSITORY="-r stable" && continue
             [ "${OPTARG}" == "vimemu" ] && INSTALL_VIMEMU="y" && continue
             [ "${OPTARG}" == "elk_stack" ] && INSTALL_ELK="y" && continue
-            [ "${OPTARG}" == "pm_stack" ] && INSTALL_PERFMON="y" && continue
+            #[ "${OPTARG}" == "pm_stack" ] && INSTALL_PERFMON="y" && continue
             [ "${OPTARG}" == "noconfigure" ] && NOCONFIGURE="y" && continue
             [ "${OPTARG}" == "showopts" ] && SHOWOPTS="y" && continue
             [ "${OPTARG}" == "daily" ] && RELEASE_DAILY="y" && continue
@@ -1307,6 +1588,7 @@ while getopts ":hy-:b:r:k:u:R:l:p:D:o:m:H:S:s:w:t:U:P:" o; do
             [ "${OPTARG}" == "nodockerbuild" ] && DOCKER_NOBUILD="y" && continue
             [ "${OPTARG}" == "nohostclient" ] && INSTALL_NOHOSTCLIENT="y" && continue
             [ "${OPTARG}" == "pullimages" ] && continue
+            [ "${OPTARG}" == "k8s_monitor" ] && K8S_MONITOR="y" && continue
             echo -e "Invalid option: '--$OPTARG'\n" >&2
             usage && exit 1
             ;;
@@ -1378,8 +1660,7 @@ if [ -z "$OSM_DEVOPS" ]; then
     fi
 fi
 
-OSM_JENKINS="$OSM_DEVOPS/jenkins"
-. $OSM_JENKINS/common/all_funcs
+. $OSM_DEVOPS/common/all_funcs
 
 [ -n "$INSTALL_LIGHTWEIGHT" ] && [ -n "$UNINSTALL" ] && uninstall_lightweight && echo -e "\nDONE" && exit 0
 [ -n "$UNINSTALL" ] && uninstall && echo -e "\nDONE" && exit 0
@@ -1387,7 +1668,7 @@ OSM_JENKINS="$OSM_DEVOPS/jenkins"
 [ -n "$UPDATE" ] && update && echo -e "\nDONE" && exit 0
 [ -n "$RECONFIGURE" ] && configure && echo -e "\nDONE" && exit 0
 [ -n "$INSTALL_ONLY" ] && [ -n "$INSTALL_ELK" ] && deploy_elk
-[ -n "$INSTALL_ONLY" ] && [ -n "$INSTALL_PERFMON" ] && deploy_perfmon
+#[ -n "$INSTALL_ONLY" ] && [ -n "$INSTALL_PERFMON" ] && deploy_perfmon
 [ -n "$INSTALL_ONLY" ] && [ -n "$INSTALL_VIMEMU" ] && install_vimemu
 [ -n "$INSTALL_ONLY" ] && echo -e "\nDONE" && exit 0