From: garciadeblas <gerardo.garciadeblas@telefonica.com>
Date: Thu, 25 Nov 2021 10:57:18 +0000 (+0100)
Subject: Fix propagation of failures in installer scripts
X-Git-Tag: v12.0.0rc1~75
X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=4d89c37a776f88b7db5719ccad964e719ab4a244;p=osm%2Fdevops.git

Fix propagation of failures in installer scripts

Change-Id: Ie83a4c9284ea63b40c37975da261634f930b6317
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
---

diff --git a/common/logging b/common/logging
index a48e01db..080eb63d 100644
--- a/common/logging
+++ b/common/logging
@@ -34,6 +34,13 @@ print_stack() {
 }
 
 
+FATAL_TRACK() {
+	echo -e "\n### $(date) ${FUNCNAME[1]}: FATAL error: $*" >&2
+	track end fatal $1 "'${@:2}'" none
+	print_stack
+	exit 1
+}
+
 FATAL() { 
 	echo -e "\n### $(date) ${FUNCNAME[1]}: FATAL error: $*" >&2
 	print_stack
diff --git a/common/track b/common/track
index 627c680d..e5a5ef76 100644
--- a/common/track
+++ b/common/track
@@ -60,13 +60,14 @@ function track(){
             operation="${1:-${osm_track_event_name}}"
             shift 1
             value="${1:-}"
+            value="${value// /_}"
             shift 1
             comment="${1:-}"
-            shift 1
             comment="${comment// /_}"
-            tags="${1:-}"
             shift 1
+            tags="${1:-}"
             tags="${tags//,/\~}"
+            shift 1
             [ "$value" == "none" ] && value=""
             [ "$comment" == "none" ] && comment=""
             [ "$tags" == "none" ] && tags=""
diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh
index 2c192597..d22bc1a0 100755
--- a/installers/full_install_osm.sh
+++ b/installers/full_install_osm.sh
@@ -607,15 +607,15 @@ EOF"
         sudo sed -i "s|Proxy.*|Proxy \"${OSM_APT_PROXY}\"; }|" ${OSM_APT_PROXY_FILE}
     fi
     sudo apt-get update || FATAL "Configured apt proxy, but couldn't run 'apt-get update'. Check ${OSM_APT_PROXY_FILE}"
-    track apt_proxy_configured
+    track prereq apt_proxy_configured_ok
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
 function install_osm() {
     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
-    track checkingroot
+    track checks checkingroot_ok
     [ "$USER" == "root" ] && FATAL "You are running the installer as root. The installer is prepared to be executed as a normal user with sudo privileges."
-    track noroot
+    track checks noroot_ok
 
     [ -z "$ASSUME_YES" ] && ! ask_user "The installation will do the following
     1. Install and configure LXD
@@ -625,7 +625,7 @@ function install_osm() {
     5. Install and initialize Kubernetes
     as pre-requirements.
     Do you want to proceed (Y/n)? " y && echo "Cancelled!" && exit 1
-    track proceed
+    track checks proceed_ok
 
     echo "Installing OSM"
 
@@ -654,32 +654,33 @@ function install_osm() {
         install_lxd
     fi
 
-    track prereqok
+    track prereq prereqok_ok
 
     if [ ! -n "$INSTALL_NODOCKER" ]; then
         DOCKER_CE_OPTS="-D ${OSM_DEVOPS} ${DEBUG_INSTALL}"
         [ -n "${DOCKER_PROXY_URL}" ] && DOCKER_CE_OPTS="${DOCKER_CE_OPTS} -p ${DOCKER_PROXY_URL}"
-        $OSM_DEVOPS/installers/install_docker_ce.sh ${DOCKER_CE_OPTS}
+        $OSM_DEVOPS/installers/install_docker_ce.sh ${DOCKER_CE_OPTS} || FATAL_TRACK docker_ce "install_docker_ce.sh failed"
     fi
 
-    track docker_ce
+    track docker_ce docker_ce_ok
 
     echo "Creating folders for installation"
     [ ! -d "$OSM_DOCKER_WORK_DIR" ] && sudo mkdir -p $OSM_DOCKER_WORK_DIR
     [ ! -d "$OSM_DOCKER_WORK_DIR/osm_pla" -a -n "$INSTALL_PLA" ] && sudo mkdir -p $OSM_DOCKER_WORK_DIR/osm_pla
     sudo cp -b $OSM_DEVOPS/installers/docker/cluster-config.yaml $OSM_DOCKER_WORK_DIR/cluster-config.yaml
 
-    $OSM_DEVOPS/installers/install_kubeadm_cluster.sh -i ${DEFAULT_IP} -d ${OSM_DOCKER_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL}
-    track k8scluster
+    $OSM_DEVOPS/installers/install_kubeadm_cluster.sh -i ${DEFAULT_IP} -d ${OSM_DOCKER_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL} || \
+    FATAL_TRACK k8scluster "install_kubeadm_cluster.sh failed"
+    track k8scluster k8scluster_ok
 
     JUJU_OPTS="-D ${OSM_DEVOPS} -s ${OSM_STACK_NAME} -i ${DEFAULT_IP} ${DEBUG_INSTALL} ${INSTALL_NOJUJU} ${INSTALL_CACHELXDIMAGES}"
     [ -n "${OSM_VCA_HOST}" ] && JUJU_OPTS="$JUJU_OPTS -H ${OSM_VCA_HOST}"
     [ -n "${LXD_CLOUD_FILE}" ] && JUJU_OPTS="$JUJU_OPTS -l ${LXD_CLOUD_FILE}"
     [ -n "${LXD_CRED_FILE}" ] && JUJU_OPTS="$JUJU_OPTS -L ${LXD_CRED_FILE}"
     [ -n "${CONTROLLER_NAME}" ] && JUJU_OPTS="$JUJU_OPTS -K ${CONTROLLER_NAME}"
-    $OSM_DEVOPS/installers/install_juju.sh ${JUJU_OPTS}
+    $OSM_DEVOPS/installers/install_juju.sh ${JUJU_OPTS} || FATAL_TRACK juju "install_juju.sh failed"
     set_vca_variables
-    track juju
+    track juju juju_ok
 
     if [ -z "$OSM_DATABASE_COMMONKEY" ]; then
         OSM_DATABASE_COMMONKEY=$(generate_secret)
@@ -688,42 +689,46 @@ function install_osm() {
 
     # Deploy OSM services
     [ -z "$DOCKER_NOBUILD" ] && generate_docker_images
-    track docker_build
+    track docker_images docker_images_ok
 
     generate_k8s_manifest_files
-    track manifest_files
+    track osm_files manifest_files_ok
     generate_docker_env_files
-    track env_files
+    track osm_files env_files_ok
 
     deploy_charmed_services
+    track deploy_osm deploy_charmed_services_ok
     kube_secrets
+    track deploy_osm kube_secrets_ok
     update_manifest_files
+    track deploy_osm update_manifest_files_ok
     namespace_vol
+    track deploy_osm namespace_vol_ok
     deploy_osm_services
+    track deploy_osm deploy_osm_services_k8s_ok
     if [ -n "$INSTALL_PLA"]; then
         # optional PLA install
         deploy_osm_pla_service
-        track deploy_osm_pla
+        track deploy_osm deploy_osm_pla_ok
     fi
-    track deploy_osm_services_k8s
     if [ -n "$INSTALL_K8S_MONITOR" ]; then
         # install OSM MONITORING
         install_k8s_monitoring
-        track install_k8s_monitoring
+        track deploy_osm install_k8s_monitoring_ok
     fi
 
     [ -z "$INSTALL_NOHOSTCLIENT" ] && install_osmclient
-    track osmclient
+    track osmclient osmclient_ok
 
     echo -e "Checking OSM health state..."
     $OSM_DEVOPS/installers/osm_health.sh -s ${OSM_STACK_NAME} -k || \
     echo -e "OSM is not healthy, but will probably converge to a healthy state soon." && \
     echo -e "Check OSM status with: kubectl -n ${OSM_STACK_NAME} get all" && \
-    track osm_unhealthy
-    track after_healthcheck
+    track healthchecks osm_unhealthy didnotconverge
+    track healthchecks after_healthcheck_ok
 
     add_local_k8scluster
-    track add_local_k8scluster
+    track final_ops add_local_k8scluster_ok
 
     wget -q -O- https://osm-download.etsi.org/ftp/osm-11.0-eleven/README2.txt &> /dev/null
     track end
@@ -790,7 +795,7 @@ function install_k8s_monitoring() {
     [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
     # install OSM monitoring
     sudo chmod +x $OSM_DEVOPS/installers/k8s/*.sh
-    sudo $OSM_DEVOPS/installers/k8s/install_osm_k8s_monitoring.sh
+    sudo $OSM_DEVOPS/installers/k8s/install_osm_k8s_monitoring.sh || FATAL_TRACK install_k8s_monitoring "k8s/install_osm_k8s_monitoring.sh failed"
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
@@ -924,6 +929,7 @@ RE_CHECK='^[a-z0-9]([-a-z0-9]*[a-z0-9])?$'
 DOCKER_REGISTRY_URL=
 DOCKER_PROXY_URL=
 MODULE_DOCKER_TAG=
+OSM_INSTALLATION_TYPE="Default"
 
 while getopts ":a:b:r:n:k:u:R:D:o:O:m:N:H:S:s:t:U:P:A:l:L:K:d:p:T:f:F:-: hy" o; do
     case "${o}" in
@@ -1051,7 +1057,7 @@ while getopts ":a:b:r:n:k:u:R:D:o:O:m:N:H:S:s:t:U:P:A:l:L:K:d:p:T:f:F:-: hy" o;
             [ "${OPTARG}" == "nohostclient" ] && INSTALL_NOHOSTCLIENT="y" && continue
             [ "${OPTARG}" == "pullimages" ] && continue
             [ "${OPTARG}" == "k8s_monitor" ] && INSTALL_K8S_MONITOR="y" && continue
-            [ "${OPTARG}" == "charmed" ] && CHARMED="y" && continue
+            [ "${OPTARG}" == "charmed" ] && CHARMED="y" && OSM_INSTALLATION_TYPE="Charmed" && continue
             [ "${OPTARG}" == "bundle" ] && continue
             [ "${OPTARG}" == "k8s" ] && continue
             [ "${OPTARG}" == "lxd" ] && continue
@@ -1098,9 +1104,12 @@ source $OSM_DEVOPS/common/all_funcs
 # Uninstall if "--uninstall"
 if [ -n "$UNINSTALL" ]; then
     if [ -n "$CHARMED" ]; then
-        ${OSM_DEVOPS}/installers/charmed_uninstall.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D $OSM_DEVOPS -t $DOCKER_TAG "$@"
+        track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none installation_type $OSM_INSTALLATION_TYPE none none
+        ${OSM_DEVOPS}/installers/charmed_uninstall.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D $OSM_DEVOPS -t $DOCKER_TAG "$@" || \
+        FATAL_TRACK charmed_uninstall "charmed_uninstall.sh failed"
     else
-        ${OSM_DEVOPS}/installers/uninstall_osm.sh "$@"
+        ${OSM_DEVOPS}/installers/uninstall_osm.sh "$@" || \
+        FATAL_TRACK community_uninstall "uninstall_osm.sh failed"
     fi
     echo -e "\nDONE"
     exit 0
@@ -1108,7 +1117,8 @@ fi
 
 # Charmed installation
 if [ -n "$CHARMED" ]; then
-    ${OSM_DEVOPS}/installers/charmed_install.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D $OSM_DEVOPS -t $DOCKER_TAG "$@"
+    ${OSM_DEVOPS}/installers/charmed_install.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D $OSM_DEVOPS -t $DOCKER_TAG "$@" || \
+    FATAL_TRACK charmed_install "charmed_install.sh failed"
     echo -e "\nDONE"
     exit 0
 fi
@@ -1172,22 +1182,11 @@ OSM_K8S_WORK_DIR="$OSM_DOCKER_WORK_DIR/osm_pods" && OSM_NAMESPACE_VOL="${OSM_HOS
 
 #Installation starts here
 wget -q -O- https://osm-download.etsi.org/ftp/osm-11.0-eleven/README.txt &> /dev/null
-OSM_TRACK_INSTALLATION_ID="$(date +%s)-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16)"
+export OSM_TRACK_INSTALLATION_ID="$(date +%s)-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16)"
 
-track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none
+track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none installation_type $OSM_INSTALLATION_TYPE none none
 
 install_osm
-echo -e "\nDONE" && exit 0
-echo -e "\nInstalling OSM from refspec: $COMMIT_ID"
-if [ -n "$INSTALL_FROM_SOURCE" ] && [ -z "$ASSUME_YES" ]; then
-    ! ask_user "The installation will take about 75-90 minutes. Continue (Y/n)? " y && echo "Cancelled!" && exit 1
-fi
-
-echo -e "Checking required packages: lxd"
-lxd --version &>/dev/null || FATAL "lxd not present, exiting."
-[ -n "$INSTALL_LXD" ] && echo -e "\nInstalling and configuring lxd" && install_lxd
-
-wget -q -O- https://osm-download.etsi.org/ftp/osm-11.0-eleven/README2.txt &> /dev/null
-track end
 echo -e "\nDONE"
+exit 0
 
diff --git a/installers/install_juju.sh b/installers/install_juju.sh
index 7181981c..cbcf506f 100755
--- a/installers/install_juju.sh
+++ b/installers/install_juju.sh
@@ -209,7 +209,7 @@ echo "OSM_DEVOPS=$OSM_DEVOPS"
 echo "HOME=$HOME"
 
 [ -z "$INSTALL_NOJUJU" ] && install_juju
-track juju_install
+track juju juju_install_ok
 
 if [ -z "$OSM_VCA_HOST" ]; then
     if [ -z "$CONTROLLER_NAME" ]; then
@@ -265,6 +265,6 @@ EOF
     [ -n "$CONTROLLER_NAME" ] && OSM_VCA_HOST=`juju show-controller $CONTROLLER_NAME |grep api-endpoints|awk -F\' '{print $2}'|awk -F\: '{print $1}'`
     [ -z "$OSM_VCA_HOST" ] && FATAL "Cannot obtain juju controller IP address"
 fi
-track juju_controller
+track juju juju_controller_ok
 
 
diff --git a/installers/install_kubeadm_cluster.sh b/installers/install_kubeadm_cluster.sh
index 1e72f372..c0f067a4 100755
--- a/installers/install_kubeadm_cluster.sh
+++ b/installers/install_kubeadm_cluster.sh
@@ -272,25 +272,25 @@ echo "HOME=$HOME"
 
 
 install_kube
-track install_k8s
+track k8scluster install_k8s_ok
 init_kubeadm $OSM_DOCKER_WORK_DIR/cluster-config.yaml
 kube_config_dir
-track init_k8s
+track k8scluster init_k8s_ok
 if [ -n "$INSTALL_K8S_MONITOR" ]; then
     # uninstall OSM MONITORING
     uninstall_k8s_monitoring
-    track uninstall_k8s_monitoring
+    track k8scluster uninstall_k8s_monitoring_ok
 fi
 #remove old namespace
 remove_k8s_namespace osm
 deploy_cni_provider
 taint_master_node
 install_helm
-track install_helm
+track k8scluster install_helm_ok
 install_k8s_storageclass
-track k8s_storageclass
+track k8scluster k8s_storageclass_ok
 install_helm_metallb
-track k8s_metallb
+track k8scluster k8s_metallb_ok
 check_for_readiness
-track k8s_ready
+track k8scluster k8s_ready_ok
 
diff --git a/installers/install_osm.sh b/installers/install_osm.sh
index d66ceb34..10e618f9 100755
--- a/installers/install_osm.sh
+++ b/installers/install_osm.sh
@@ -131,7 +131,6 @@ EOF"
         sudo sed -i "s|Proxy.*|Proxy \"${OSM_APT_PROXY}\"; }|" ${OSM_APT_PROXY_FILE}
     fi
     sudo apt-get update || FATAL "Configured apt proxy, but couldn't run 'apt-get update'. Check ${OSM_APT_PROXY_FILE}"
-    track apt_proxy_configured
     [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
 }
 
diff --git a/installers/test_track.sh b/installers/test_track.sh
index 8db0260e..65e6e86f 100755
--- a/installers/test_track.sh
+++ b/installers/test_track.sh
@@ -14,21 +14,40 @@
 #
 
 source ../common/track
+source ../common/logging
 
-RELEASE="ReleaseTEN"
+RELEASE="test_track"
 OSM_DOCKER_TAG=latest
 OSM_TRACK_INSTALLATION_ID="$(date +%s)-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16)"
 
+echo "Next track call should fail"
+echo "track"
 track
-track start
+echo
+echo "Next, several track calls are done with different args" 
+echo "track test-event"
+track test-event
 sleep 1
-track start release $RELEASE
+echo "track test-event release $RELEASE"
+track test-event release $RELEASE
 sleep 1
-track start docker_tag $OSM_DOCKER_TAG none none
+echo "track test-event docker_tag $OSM_DOCKER_TAG none none"
+track test-event docker_tag $OSM_DOCKER_TAG none none
 sleep 1
-track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none
+echo "track test-event release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none"
+track test-event release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none
 sleep 1
-track my-event my-op my-value "My comment" "tag1,tag2"
+echo 'track test-event my-op my-value "My comment" none'
+track test-event my-op my-value "My comment" none
 sleep 1
-track my-second-event op1 value1 "My comment1 on second event" none op2 value2 "My comment2 on second event" none
+echo 'track test-event my-op my-value "My comment" "tag1,tag2"'
+track test-event my-op my-value "My comment" "tag1,tag2"
+sleep 1
+echo 'track test-event op1 value1 "My comment1 on test event" none op2 value2 "My comment2 on test event" none'
+track test-event op1 value1 "My comment1 on test event" none op2 value2 "My comment2 on test event" none
+sleep 1
+echo
+echo "Next track call will be done from function FATAL_TRACK"
+echo 'FATAL_TRACK test-event "Fatal error during execution"'
+FATAL_TRACK test-event "Fatal error during execution"