Feature 11007: replace charmed mongodb with bitnami helm chart
[osm/devops.git] / installers / full_install_osm.sh
index 23d8f07..e053bd7 100755 (executable)
@@ -16,7 +16,7 @@
 function usage(){
     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
     echo -e "usage: $0 [OPTIONS]"
-    echo -e "Install OSM from binaries or source code (by default, from binaries)"
+    echo -e "Install OSM"
     echo -e "  OPTIONS"
     echo -e "     -h / --help:    print this help"
     echo -e "     -y:             do not prompt for confirmation, assumes yes"
@@ -180,16 +180,15 @@ function install_osmclient(){
     key_location=$CLIENT_REPOSITORY_BASE/$CLIENT_RELEASE/$CLIENT_REPOSITORY_KEY
     curl $key_location | sudo APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add -
     sudo add-apt-repository -y "deb [arch=amd64] $CLIENT_REPOSITORY_BASE/$CLIENT_RELEASE $CLIENT_REPOSITORY osmclient IM"
-    sudo apt-get update
-    sudo apt-get install -y python3-pip
+    sudo apt-get -y update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y python3-pip
     sudo -H LC_ALL=C python3 -m pip install -U pip
-    sudo -H LC_ALL=C python3 -m pip install -U python-magic pyangbind verboselogs
-    sudo apt-get install -y python3-osm-im python3-osmclient
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y python3-osm-im python3-osmclient
     if [ -f /usr/lib/python3/dist-packages/osm_im/requirements.txt ]; then
         python3 -m pip install -r /usr/lib/python3/dist-packages/osm_im/requirements.txt
     fi
     if [ -f /usr/lib/python3/dist-packages/osmclient/requirements.txt ]; then
-        sudo apt-get install -y libcurl4-openssl-dev libssl-dev libmagic1
+        sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libmagic1
         python3 -m pip install -r /usr/lib/python3/dist-packages/osmclient/requirements.txt
     fi
     [ -z "$INSTALL_LIGHTWEIGHT" ] && export OSM_HOSTNAME=`lxc list | awk '($2=="SO-ub"){print $6}'`
@@ -216,62 +215,6 @@ function docker_login() {
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
-function pull_docker_images() {
-    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
-    echo "Pulling docker images"
-    [ -n "${DOCKER_REGISTRY_URL}" ] && docker_login
-
-    echo "Pulling non-OSM docker images"
-    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q KAFKA ; then
-        sg docker -c "docker pull wurstmeister/zookeeper" || FATAL "cannot get zookeeper docker image"
-        sg docker -c "docker pull wurstmeister/kafka:${KAFKA_TAG}" || FATAL "cannot get kafka docker image"
-    fi
-
-    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q PROMETHEUS ; then
-        sg docker -c "docker pull prom/prometheus:${PROMETHEUS_TAG}" || FATAL "cannot get prometheus docker image"
-    fi
-
-    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q PROMETHEUS-CADVISOR ; then
-        sg docker -c "docker pull google/cadvisor:${PROMETHEUS_CADVISOR_TAG}" || FATAL "cannot get prometheus cadvisor docker image"
-    fi
-
-    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q GRAFANA ; then
-        sg docker -c "docker pull grafana/grafana:${GRAFANA_TAG}" || FATAL "cannot get grafana docker image"
-        sg docker -c "docker pull kiwigrid/k8s-sidecar:${KIWIGRID_K8S_SIDECAR_TAG}" || FATAL "cannot get kiwigrid k8s-sidecar docker image"
-    fi
-
-    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q NBI || echo $TO_REBUILD | grep -q KEYSTONE-DB ; then
-        sg docker -c "docker pull mariadb:${KEYSTONEDB_TAG}" || FATAL "cannot get keystone-db docker image"
-    fi
-
-    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q RO ; then
-        sg docker -c "docker pull mysql:5" || FATAL "cannot get mysql docker image"
-    fi
-
-    list_of_modules="MON NBI KEYSTONE RO LCM NG-UI osmclient"
-    if [ -n "$INSTALL_NGSA" ]; then
-        list_of_modules="${list_of_modules} Airflow Webhook"
-    else
-        list_of_modules="${list_of_modules} POL"
-    fi
-    if [ -n "$INSTALL_PLA" ]; then
-        list_of_modules="${list_of_modules} PLA"
-    fi
-    echo "Pulling OSM docker images for the following modules: ${list_of_modules}"
-    for module in ${list_of_modules}; do
-        module_lower=${module,,}
-        module_tag="${OSM_DOCKER_TAG}"
-        if [ -n "${MODULE_DOCKER_TAG}" ] && echo $TO_REBUILD | grep -q $module ; then
-            module_tag="${MODULE_DOCKER_TAG}"
-        fi
-        echo "Pulling ${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module_lower}:${module_tag} docker image"
-        sg docker -c "docker pull ${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module_lower}:${module_tag}" || FATAL "cannot pull $module docker image"
-    done
-
-    echo "Finished pulling docker images"
-    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
-}
-
 function generate_docker_images() {
     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
     echo "Generating docker images"
@@ -333,13 +276,18 @@ EOF"
     # OSM_HELM_OPTS="${OSM_HELM_OPTS} --set nbi.useOsmSecret=false"
     OSM_HELM_OPTS="${OSM_HELM_OPTS} --set global.image.repositoryBase=${DOCKER_REGISTRY_URL}${DOCKER_USER}"
     OSM_HELM_OPTS="${OSM_HELM_OPTS} --set mysql.dbHostPath=${OSM_NAMESPACE_VOL}"
-    OSM_HELM_OPTS="${OSM_HELM_OPTS} --set mon.config.OS_NOTIFIER_URI=http://${OSM_DEFAULT_IP}:8662"
     OSM_HELM_OPTS="${OSM_HELM_OPTS} --set vca.host=${OSM_VCA_HOST}"
     OSM_HELM_OPTS="${OSM_HELM_OPTS} --set vca.secret=${OSM_VCA_SECRET}"
     OSM_HELM_OPTS="${OSM_HELM_OPTS} --set vca.cacert=${OSM_VCA_CACERT}"
     [ -n "$OSM_VCA_APIPROXY" ] && OSM_HELM_OPTS="${OSM_HELM_OPTS} --set lcm.config.OSMLCM_VCA_APIPROXY=${OSM_VCA_APIPROXY}"
-    [ ! "$OSM_DOCKER_TAG" == "13" ] && OSM_HELM_OPTS="${OSM_HELM_OPTS} --set global.image.tag=${OSM_DOCKER_TAG}"
+    [ ! "$OSM_DOCKER_TAG" == "testing-daily" ] && OSM_HELM_OPTS="${OSM_HELM_OPTS} --set global.image.tag=${OSM_DOCKER_TAG}"
     [ -n "${INSTALL_NGSA}" ] || OSM_HELM_OPTS="${OSM_HELM_OPTS} --set global.oldServiceAssurance=true"
+    if [ -n "${OSM_BEHIND_PROXY}" ]; then
+        OSM_HELM_OPTS="${OSM_HELM_OPTS} --set global.behindHttpProxy=true"
+        [ -n "${HTTP_PROXY}" ] && OSM_HELM_OPTS="${OSM_HELM_OPTS} --set global.httpProxy.HTTP_PROXY=${HTTP_PROXY}"
+        [ -n "${HTTPS_PROXY}" ] && OSM_HELM_OPTS="${OSM_HELM_OPTS} --set global.httpProxy.HTTPS_PROXY=${HTTPS_PROXY}"
+        [ -n "${NO_PROXY}" ] && OSM_HELM_OPTS="${OSM_HELM_OPTS} --set global.httpProxy.NO_PROXY=${NO_PROXY}"
+    fi
 
     echo "helm -n $OSM_NAMESPACE install $OSM_NAMESPACE $OSM_DEVOPS/installers/helm/osm -f ${OSM_HELM_WORK_DIR}/osm-values.yaml ${OSM_HELM_OPTS}"
     helm -n $OSM_NAMESPACE install $OSM_NAMESPACE $OSM_DEVOPS/installers/helm/osm -f ${OSM_HELM_WORK_DIR}/osm-values.yaml ${OSM_HELM_OPTS}
@@ -357,10 +305,13 @@ function deploy_charmed_services() {
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
-function deploy_osm_pla_service() {
+#deploy mongodb
+function deploy_mongodb() {
     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
-    # corresponding to deploy_osm_services
-    kubectl apply -n $OSM_NAMESPACE -f $OSM_DOCKER_WORK_DIR/osm_pla
+    MONGO_OPTS="-d ${OSM_HELM_WORK_DIR} -D ${OSM_DEVOPS} -t ${OSM_DOCKER_TAG} -U ${DOCKER_USER} ${DEBUG_INSTALL}"
+    [ -n "${DOCKER_REGISTRY_URL}" ] && MONGO_OPTS="${MONGO_OPTS} -r ${DOCKER_REGISTRY_URL}"
+    $OSM_DEVOPS/installers/install_mongodb.sh ${MONGO_OPTS} || \
+    FATAL_TRACK install_osm_mongodb_service "install_mongodb.sh failed"
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
@@ -486,29 +437,24 @@ function install_osm() {
 
     trap ctrl_c INT
 
-    # TODO: move this under start
-    [ -n "$DOCKER_REGISTRY_URL" ] && parse_docker_registry_url
 
     check_osm_behind_proxy
-    track checks proxy_ok
-
     check_packages "git wget curl tar snapd"
-
     sudo snap install jq || FATAL "Could not install jq (snap package). Make sure that snap works"
-
     find_devops_folder
 
-    track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none installation_type $OSM_INSTALLATION_TYPE none none
+    track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none installation_type $OSM_INSTALLATION_TYPE none none os_info $os_info none none
 
     track checks checkingroot_ok
     [ "$USER" == "root" ] && FATAL "You are running the installer as root. The installer is prepared to be executed as a normal user with sudo privileges."
     track checks noroot_ok
-
     ask_proceed
     track checks proceed_ok
 
     echo "Installing OSM"
 
+    [ -n "$DOCKER_REGISTRY_URL" ] && parse_docker_registry_url
+
     echo "Determining IP address of the interface with the default route"
     [ -z "$OSM_DEFAULT_IF" ] && OSM_DEFAULT_IF=$(ip route list|awk '$1=="default" {print $5; exit}')
     [ -z "$OSM_DEFAULT_IF" ] && OSM_DEFAULT_IF=$(route -n |awk '$1~/^0.0.0.0/ {print $8; exit}')
@@ -538,11 +484,10 @@ function install_osm() {
     track docker_ce docker_ce_ok
 
     echo "Creating folders for installation"
-    [ ! -d "$OSM_DOCKER_WORK_DIR" ] && sudo mkdir -p $OSM_DOCKER_WORK_DIR
-    [ ! -d "$OSM_DOCKER_WORK_DIR/osm_pla" -a -n "$INSTALL_PLA" ] && sudo mkdir -p $OSM_DOCKER_WORK_DIR/osm_pla
-    sudo cp -b $OSM_DEVOPS/installers/docker/cluster-config.yaml $OSM_DOCKER_WORK_DIR/cluster-config.yaml
+    [ ! -d "$OSM_WORK_DIR" ] && sudo mkdir -p $OSM_WORK_DIR
+    sudo cp -b $OSM_DEVOPS/installers/kubeadm-config.yaml $OSM_WORK_DIR/kubeadm-config.yaml
 
-    $OSM_DEVOPS/installers/install_kubeadm_cluster.sh -i ${OSM_DEFAULT_IP} -d ${OSM_DOCKER_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL} || \
+    $OSM_DEVOPS/installers/install_kubeadm_cluster.sh -i ${OSM_DEFAULT_IP} -d ${OSM_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL} || \
     FATAL_TRACK k8scluster "install_kubeadm_cluster.sh failed"
     track k8scluster k8scluster_ok
 
@@ -557,12 +502,11 @@ function install_osm() {
     track juju juju_ok
 
     # Deploy OSM services
-    [ -z "$DOCKER_NOBUILD" ] && pull_docker_images
     [ -z "$DOCKER_NOBUILD" ] && [ -z "$PULL_IMAGES" ] && generate_docker_images
     track docker_images docker_images_ok
 
-    deploy_charmed_services
-    track deploy_osm deploy_charmed_services_ok
+    deploy_mongodb
+    track deploy_osm deploy_mongodb_ok
     deploy_osm_services
     track deploy_osm deploy_osm_services_k8s_ok
     if [ -n "$INSTALL_K8S_MONITOR" ]; then
@@ -591,7 +535,7 @@ function install_osm() {
 
     arrange_docker_default_network_policy
 
-    wget -q -O- https://osm-download.etsi.org/ftp/osm-13.0-thirteen/README2.txt &> /dev/null
+    wget -q -O- https://osm-download.etsi.org/ftp/osm-14.0-fourteen/README2.txt &> /dev/null
     track end
     sudo find /etc/osm
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
@@ -698,9 +642,7 @@ function dump_vars(){
     echo "OPENSTACK_VM_NAME"="$OPENSTACK_VM_NAME"
     echo "OSM_DEVOPS=$OSM_DEVOPS"
     echo "OSM_DOCKER_TAG=$OSM_DOCKER_TAG"
-    echo "OSM_DOCKER_WORK_DIR=$OSM_DOCKER_WORK_DIR"
     echo "OSM_HELM_WORK_DIR=$OSM_HELM_WORK_DIR"
-    echo "OSM_K8S_WORK_DIR=$OSM_K8S_WORK_DIR"
     echo "OSM_NAMESPACE=$OSM_NAMESPACE"
     echo "OSM_VCA_HOST=$OSM_VCA_HOST"
     echo "OSM_VCA_PUBKEY=$OSM_VCA_PUBKEY"
@@ -784,12 +726,10 @@ DOCKER_NOBUILD=""
 REPOSITORY_KEY="OSM%20ETSI%20Release%20Key.gpg"
 REPOSITORY_BASE="https://osm-download.etsi.org/repository/osm/debian"
 OSM_WORK_DIR="/etc/osm"
-OSM_DOCKER_WORK_DIR="${OSM_WORK_DIR}/docker"
-OSM_K8S_WORK_DIR="${OSM_DOCKER_WORK_DIR}/osm_pods"
 OSM_HELM_WORK_DIR="${OSM_WORK_DIR}/helm"
 OSM_HOST_VOL="/var/lib/osm"
 OSM_NAMESPACE_VOL="${OSM_HOST_VOL}/${OSM_NAMESPACE}"
-OSM_DOCKER_TAG=latest
+OSM_DOCKER_TAG="testing-daily"
 DOCKER_USER=opensourcemano
 PULL_IMAGES="y"
 KAFKA_TAG=2.11-1.0.2
@@ -997,15 +937,26 @@ if [ -n "$UNINSTALL" ]; then
     exit 0
 fi
 
+# Installation starts here
+
+# Get README and create OSM_TRACK_INSTALLATION_ID
+wget -q -O- https://osm-download.etsi.org/ftp/osm-14.0-fourteen/README.txt &> /dev/null
+export OSM_TRACK_INSTALLATION_ID="$(date +%s)-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16)"
+
+# Get OS info to be tracked
+os_distro=$(lsb_release -i 2>/dev/null | awk '{print $3}')
+echo $os_distro
+os_release=$(lsb_release -r 2>/dev/null | awk '{print $2}')
+echo $os_release
+os_info="${os_distro}_${os_release}"
+os_info="${os_info// /_}"
+
 # Charmed installation
 if [ -n "$CHARMED" ]; then
     sudo snap install jq || FATAL "Could not install jq (snap package). Make sure that snap works"
-    export OSM_TRACK_INSTALLATION_ID="$(date +%s)-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16)"
-    track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none installation_type $OSM_INSTALLATION_TYPE none none
     ${OSM_DEVOPS}/installers/charmed_install.sh --tag $OSM_DOCKER_TAG "$@" || \
     FATAL_TRACK charmed_install "charmed_install.sh failed"
-    wget -q -O- https://osm-download.etsi.org/ftp/osm-13.0-thirteen/README2.txt &> /dev/null
-    track end installation_type $OSM_INSTALLATION_TYPE
+    wget -q -O- https://osm-download.etsi.org/ftp/osm-14.0-fourteen/README2.txt &> /dev/null
     echo -e "\nDONE"
     exit 0
 fi
@@ -1018,19 +969,16 @@ if [ -n "$INSTALL_TO_OPENSTACK" ]; then
 fi
 
 # Community_installer
-
+# Check incompatible options
 [ -n "$TO_REBUILD" ] && [ "$TO_REBUILD" != " NONE" ] && echo $TO_REBUILD | grep -q NONE && FATAL "Incompatible option: -m NONE cannot be used with other -m options"
 [ -n "$TO_REBUILD" ] && [ "$TO_REBUILD" == " PLA" ] && [ -z "$INSTALL_PLA" ] && FATAL "Incompatible option: -m PLA cannot be used without --pla option"
+# Special cases go first
 # if develop, we force master
 [ -z "$COMMIT_ID" ] && [ -n "$DEVELOP" ] && COMMIT_ID="master"
-OSM_K8S_WORK_DIR="$OSM_DOCKER_WORK_DIR/osm_pods" && OSM_NAMESPACE_VOL="${OSM_HOST_VOL}/${OSM_NAMESPACE}"
 [ -n "$INSTALL_ONLY" ] && [ -n "$INSTALL_K8S_MONITOR" ] && install_k8s_monitoring
 [ -n "$INSTALL_ONLY" ] && [ -n "$INSTALL_NGSA" ] && install_osm_ngsa_service
 [ -n "$INSTALL_ONLY" ] && echo -e "\nDONE" && exit 0
-
-#Installation starts here
-wget -q -O- https://osm-download.etsi.org/ftp/osm-13.0-thirteen/README.txt &> /dev/null
-export OSM_TRACK_INSTALLATION_ID="$(date +%s)-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16)"
+# This is where installation starts
 install_osm
 echo -e "\nDONE"
 exit 0