From c0d0234449b80a56febe1f5a933196643a13e463 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Thu, 25 Nov 2021 11:57:18 +0100 Subject: [PATCH] Fix propagation of failures in installer scripts Change-Id: Ie83a4c9284ea63b40c37975da261634f930b6317 Signed-off-by: garciadeblas --- common/logging | 7 +++ common/track | 5 +- installers/full_install_osm.sh | 79 +++++++++++++-------------- installers/install_juju.sh | 4 +- installers/install_kubeadm_cluster.sh | 14 ++--- installers/install_osm.sh | 1 - installers/test_track.sh | 33 ++++++++--- 7 files changed, 84 insertions(+), 59 deletions(-) diff --git a/common/logging b/common/logging index a48e01db..080eb63d 100644 --- a/common/logging +++ b/common/logging @@ -34,6 +34,13 @@ print_stack() { } +FATAL_TRACK() { + echo -e "\n### $(date) ${FUNCNAME[1]}: FATAL error: $*" >&2 + track end fatal $1 "'${@:2}'" none + print_stack + exit 1 +} + FATAL() { echo -e "\n### $(date) ${FUNCNAME[1]}: FATAL error: $*" >&2 print_stack diff --git a/common/track b/common/track index 627c680d..e5a5ef76 100644 --- a/common/track +++ b/common/track @@ -60,13 +60,14 @@ function track(){ operation="${1:-${osm_track_event_name}}" shift 1 value="${1:-}" + value="${value// /_}" shift 1 comment="${1:-}" - shift 1 comment="${comment// /_}" - tags="${1:-}" shift 1 + tags="${1:-}" tags="${tags//,/\~}" + shift 1 [ "$value" == "none" ] && value="" [ "$comment" == "none" ] && comment="" [ "$tags" == "none" ] && tags="" diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh index 2c192597..d22bc1a0 100755 --- a/installers/full_install_osm.sh +++ b/installers/full_install_osm.sh @@ -607,15 +607,15 @@ EOF" sudo sed -i "s|Proxy.*|Proxy \"${OSM_APT_PROXY}\"; }|" ${OSM_APT_PROXY_FILE} fi sudo apt-get update || FATAL "Configured apt proxy, but couldn't run 'apt-get update'. Check ${OSM_APT_PROXY_FILE}" - track apt_proxy_configured + track prereq apt_proxy_configured_ok [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } function install_osm() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function - track checkingroot + track checks checkingroot_ok [ "$USER" == "root" ] && FATAL "You are running the installer as root. The installer is prepared to be executed as a normal user with sudo privileges." - track noroot + track checks noroot_ok [ -z "$ASSUME_YES" ] && ! ask_user "The installation will do the following 1. Install and configure LXD @@ -625,7 +625,7 @@ function install_osm() { 5. Install and initialize Kubernetes as pre-requirements. Do you want to proceed (Y/n)? " y && echo "Cancelled!" && exit 1 - track proceed + track checks proceed_ok echo "Installing OSM" @@ -654,32 +654,33 @@ function install_osm() { install_lxd fi - track prereqok + track prereq prereqok_ok if [ ! -n "$INSTALL_NODOCKER" ]; then DOCKER_CE_OPTS="-D ${OSM_DEVOPS} ${DEBUG_INSTALL}" [ -n "${DOCKER_PROXY_URL}" ] && DOCKER_CE_OPTS="${DOCKER_CE_OPTS} -p ${DOCKER_PROXY_URL}" - $OSM_DEVOPS/installers/install_docker_ce.sh ${DOCKER_CE_OPTS} + $OSM_DEVOPS/installers/install_docker_ce.sh ${DOCKER_CE_OPTS} || FATAL_TRACK docker_ce "install_docker_ce.sh failed" fi - track docker_ce + track docker_ce docker_ce_ok echo "Creating folders for installation" [ ! -d "$OSM_DOCKER_WORK_DIR" ] && sudo mkdir -p $OSM_DOCKER_WORK_DIR [ ! -d "$OSM_DOCKER_WORK_DIR/osm_pla" -a -n "$INSTALL_PLA" ] && sudo mkdir -p $OSM_DOCKER_WORK_DIR/osm_pla sudo cp -b $OSM_DEVOPS/installers/docker/cluster-config.yaml $OSM_DOCKER_WORK_DIR/cluster-config.yaml - $OSM_DEVOPS/installers/install_kubeadm_cluster.sh -i ${DEFAULT_IP} -d ${OSM_DOCKER_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL} - track k8scluster + $OSM_DEVOPS/installers/install_kubeadm_cluster.sh -i ${DEFAULT_IP} -d ${OSM_DOCKER_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL} || \ + FATAL_TRACK k8scluster "install_kubeadm_cluster.sh failed" + track k8scluster k8scluster_ok JUJU_OPTS="-D ${OSM_DEVOPS} -s ${OSM_STACK_NAME} -i ${DEFAULT_IP} ${DEBUG_INSTALL} ${INSTALL_NOJUJU} ${INSTALL_CACHELXDIMAGES}" [ -n "${OSM_VCA_HOST}" ] && JUJU_OPTS="$JUJU_OPTS -H ${OSM_VCA_HOST}" [ -n "${LXD_CLOUD_FILE}" ] && JUJU_OPTS="$JUJU_OPTS -l ${LXD_CLOUD_FILE}" [ -n "${LXD_CRED_FILE}" ] && JUJU_OPTS="$JUJU_OPTS -L ${LXD_CRED_FILE}" [ -n "${CONTROLLER_NAME}" ] && JUJU_OPTS="$JUJU_OPTS -K ${CONTROLLER_NAME}" - $OSM_DEVOPS/installers/install_juju.sh ${JUJU_OPTS} + $OSM_DEVOPS/installers/install_juju.sh ${JUJU_OPTS} || FATAL_TRACK juju "install_juju.sh failed" set_vca_variables - track juju + track juju juju_ok if [ -z "$OSM_DATABASE_COMMONKEY" ]; then OSM_DATABASE_COMMONKEY=$(generate_secret) @@ -688,42 +689,46 @@ function install_osm() { # Deploy OSM services [ -z "$DOCKER_NOBUILD" ] && generate_docker_images - track docker_build + track docker_images docker_images_ok generate_k8s_manifest_files - track manifest_files + track osm_files manifest_files_ok generate_docker_env_files - track env_files + track osm_files env_files_ok deploy_charmed_services + track deploy_osm deploy_charmed_services_ok kube_secrets + track deploy_osm kube_secrets_ok update_manifest_files + track deploy_osm update_manifest_files_ok namespace_vol + track deploy_osm namespace_vol_ok deploy_osm_services + track deploy_osm deploy_osm_services_k8s_ok if [ -n "$INSTALL_PLA"]; then # optional PLA install deploy_osm_pla_service - track deploy_osm_pla + track deploy_osm deploy_osm_pla_ok fi - track deploy_osm_services_k8s if [ -n "$INSTALL_K8S_MONITOR" ]; then # install OSM MONITORING install_k8s_monitoring - track install_k8s_monitoring + track deploy_osm install_k8s_monitoring_ok fi [ -z "$INSTALL_NOHOSTCLIENT" ] && install_osmclient - track osmclient + track osmclient osmclient_ok echo -e "Checking OSM health state..." $OSM_DEVOPS/installers/osm_health.sh -s ${OSM_STACK_NAME} -k || \ echo -e "OSM is not healthy, but will probably converge to a healthy state soon." && \ echo -e "Check OSM status with: kubectl -n ${OSM_STACK_NAME} get all" && \ - track osm_unhealthy - track after_healthcheck + track healthchecks osm_unhealthy didnotconverge + track healthchecks after_healthcheck_ok add_local_k8scluster - track add_local_k8scluster + track final_ops add_local_k8scluster_ok wget -q -O- https://osm-download.etsi.org/ftp/osm-11.0-eleven/README2.txt &> /dev/null track end @@ -790,7 +795,7 @@ function install_k8s_monitoring() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function # install OSM monitoring sudo chmod +x $OSM_DEVOPS/installers/k8s/*.sh - sudo $OSM_DEVOPS/installers/k8s/install_osm_k8s_monitoring.sh + sudo $OSM_DEVOPS/installers/k8s/install_osm_k8s_monitoring.sh || FATAL_TRACK install_k8s_monitoring "k8s/install_osm_k8s_monitoring.sh failed" [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } @@ -924,6 +929,7 @@ RE_CHECK='^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' DOCKER_REGISTRY_URL= DOCKER_PROXY_URL= MODULE_DOCKER_TAG= +OSM_INSTALLATION_TYPE="Default" while getopts ":a:b:r:n:k:u:R:D:o:O:m:N:H:S:s:t:U:P:A:l:L:K:d:p:T:f:F:-: hy" o; do case "${o}" in @@ -1051,7 +1057,7 @@ while getopts ":a:b:r:n:k:u:R:D:o:O:m:N:H:S:s:t:U:P:A:l:L:K:d:p:T:f:F:-: hy" o; [ "${OPTARG}" == "nohostclient" ] && INSTALL_NOHOSTCLIENT="y" && continue [ "${OPTARG}" == "pullimages" ] && continue [ "${OPTARG}" == "k8s_monitor" ] && INSTALL_K8S_MONITOR="y" && continue - [ "${OPTARG}" == "charmed" ] && CHARMED="y" && continue + [ "${OPTARG}" == "charmed" ] && CHARMED="y" && OSM_INSTALLATION_TYPE="Charmed" && continue [ "${OPTARG}" == "bundle" ] && continue [ "${OPTARG}" == "k8s" ] && continue [ "${OPTARG}" == "lxd" ] && continue @@ -1098,9 +1104,12 @@ source $OSM_DEVOPS/common/all_funcs # Uninstall if "--uninstall" if [ -n "$UNINSTALL" ]; then if [ -n "$CHARMED" ]; then - ${OSM_DEVOPS}/installers/charmed_uninstall.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D $OSM_DEVOPS -t $DOCKER_TAG "$@" + track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none installation_type $OSM_INSTALLATION_TYPE none none + ${OSM_DEVOPS}/installers/charmed_uninstall.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D $OSM_DEVOPS -t $DOCKER_TAG "$@" || \ + FATAL_TRACK charmed_uninstall "charmed_uninstall.sh failed" else - ${OSM_DEVOPS}/installers/uninstall_osm.sh "$@" + ${OSM_DEVOPS}/installers/uninstall_osm.sh "$@" || \ + FATAL_TRACK community_uninstall "uninstall_osm.sh failed" fi echo -e "\nDONE" exit 0 @@ -1108,7 +1117,8 @@ fi # Charmed installation if [ -n "$CHARMED" ]; then - ${OSM_DEVOPS}/installers/charmed_install.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D $OSM_DEVOPS -t $DOCKER_TAG "$@" + ${OSM_DEVOPS}/installers/charmed_install.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D $OSM_DEVOPS -t $DOCKER_TAG "$@" || \ + FATAL_TRACK charmed_install "charmed_install.sh failed" echo -e "\nDONE" exit 0 fi @@ -1172,22 +1182,11 @@ OSM_K8S_WORK_DIR="$OSM_DOCKER_WORK_DIR/osm_pods" && OSM_NAMESPACE_VOL="${OSM_HOS #Installation starts here wget -q -O- https://osm-download.etsi.org/ftp/osm-11.0-eleven/README.txt &> /dev/null -OSM_TRACK_INSTALLATION_ID="$(date +%s)-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16)" +export OSM_TRACK_INSTALLATION_ID="$(date +%s)-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16)" -track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none +track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none installation_type $OSM_INSTALLATION_TYPE none none install_osm -echo -e "\nDONE" && exit 0 -echo -e "\nInstalling OSM from refspec: $COMMIT_ID" -if [ -n "$INSTALL_FROM_SOURCE" ] && [ -z "$ASSUME_YES" ]; then - ! ask_user "The installation will take about 75-90 minutes. Continue (Y/n)? " y && echo "Cancelled!" && exit 1 -fi - -echo -e "Checking required packages: lxd" -lxd --version &>/dev/null || FATAL "lxd not present, exiting." -[ -n "$INSTALL_LXD" ] && echo -e "\nInstalling and configuring lxd" && install_lxd - -wget -q -O- https://osm-download.etsi.org/ftp/osm-11.0-eleven/README2.txt &> /dev/null -track end echo -e "\nDONE" +exit 0 diff --git a/installers/install_juju.sh b/installers/install_juju.sh index 7181981c..cbcf506f 100755 --- a/installers/install_juju.sh +++ b/installers/install_juju.sh @@ -209,7 +209,7 @@ echo "OSM_DEVOPS=$OSM_DEVOPS" echo "HOME=$HOME" [ -z "$INSTALL_NOJUJU" ] && install_juju -track juju_install +track juju juju_install_ok if [ -z "$OSM_VCA_HOST" ]; then if [ -z "$CONTROLLER_NAME" ]; then @@ -265,6 +265,6 @@ EOF [ -n "$CONTROLLER_NAME" ] && OSM_VCA_HOST=`juju show-controller $CONTROLLER_NAME |grep api-endpoints|awk -F\' '{print $2}'|awk -F\: '{print $1}'` [ -z "$OSM_VCA_HOST" ] && FATAL "Cannot obtain juju controller IP address" fi -track juju_controller +track juju juju_controller_ok diff --git a/installers/install_kubeadm_cluster.sh b/installers/install_kubeadm_cluster.sh index 1e72f372..c0f067a4 100755 --- a/installers/install_kubeadm_cluster.sh +++ b/installers/install_kubeadm_cluster.sh @@ -272,25 +272,25 @@ echo "HOME=$HOME" install_kube -track install_k8s +track k8scluster install_k8s_ok init_kubeadm $OSM_DOCKER_WORK_DIR/cluster-config.yaml kube_config_dir -track init_k8s +track k8scluster init_k8s_ok if [ -n "$INSTALL_K8S_MONITOR" ]; then # uninstall OSM MONITORING uninstall_k8s_monitoring - track uninstall_k8s_monitoring + track k8scluster uninstall_k8s_monitoring_ok fi #remove old namespace remove_k8s_namespace osm deploy_cni_provider taint_master_node install_helm -track install_helm +track k8scluster install_helm_ok install_k8s_storageclass -track k8s_storageclass +track k8scluster k8s_storageclass_ok install_helm_metallb -track k8s_metallb +track k8scluster k8s_metallb_ok check_for_readiness -track k8s_ready +track k8scluster k8s_ready_ok diff --git a/installers/install_osm.sh b/installers/install_osm.sh index d66ceb34..10e618f9 100755 --- a/installers/install_osm.sh +++ b/installers/install_osm.sh @@ -131,7 +131,6 @@ EOF" sudo sed -i "s|Proxy.*|Proxy \"${OSM_APT_PROXY}\"; }|" ${OSM_APT_PROXY_FILE} fi sudo apt-get update || FATAL "Configured apt proxy, but couldn't run 'apt-get update'. Check ${OSM_APT_PROXY_FILE}" - track apt_proxy_configured [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } diff --git a/installers/test_track.sh b/installers/test_track.sh index 8db0260e..65e6e86f 100755 --- a/installers/test_track.sh +++ b/installers/test_track.sh @@ -14,21 +14,40 @@ # source ../common/track +source ../common/logging -RELEASE="ReleaseTEN" +RELEASE="test_track" OSM_DOCKER_TAG=latest OSM_TRACK_INSTALLATION_ID="$(date +%s)-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 16)" +echo "Next track call should fail" +echo "track" track -track start +echo +echo "Next, several track calls are done with different args" +echo "track test-event" +track test-event sleep 1 -track start release $RELEASE +echo "track test-event release $RELEASE" +track test-event release $RELEASE sleep 1 -track start docker_tag $OSM_DOCKER_TAG none none +echo "track test-event docker_tag $OSM_DOCKER_TAG none none" +track test-event docker_tag $OSM_DOCKER_TAG none none sleep 1 -track start release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none +echo "track test-event release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none" +track test-event release $RELEASE none none docker_tag $OSM_DOCKER_TAG none none sleep 1 -track my-event my-op my-value "My comment" "tag1,tag2" +echo 'track test-event my-op my-value "My comment" none' +track test-event my-op my-value "My comment" none sleep 1 -track my-second-event op1 value1 "My comment1 on second event" none op2 value2 "My comment2 on second event" none +echo 'track test-event my-op my-value "My comment" "tag1,tag2"' +track test-event my-op my-value "My comment" "tag1,tag2" +sleep 1 +echo 'track test-event op1 value1 "My comment1 on test event" none op2 value2 "My comment2 on test event" none' +track test-event op1 value1 "My comment1 on test event" none op2 value2 "My comment2 on test event" none +sleep 1 +echo +echo "Next track call will be done from function FATAL_TRACK" +echo 'FATAL_TRACK test-event "Fatal error during execution"' +FATAL_TRACK test-event "Fatal error during execution" -- 2.25.1