Merge "feature(prometheus): Configuration can be dynamically replaced"
authorbeierlm <mark.beierl@canonical.com>
Thu, 30 Apr 2020 18:52:01 +0000 (20:52 +0200)
committerGerrit Code Review <root@osm.etsi.org>
Thu, 30 Apr 2020 18:52:01 +0000 (20:52 +0200)
1  2 
installers/full_install_osm.sh

@@@ -35,13 -35,12 +35,13 @@@ function usage()
      echo -e "     -A <VCA apiproxy> use VCA/juju API proxy"
      echo -e "     --vimemu:       additionally deploy the VIM emulator as a docker container"
      echo -e "     --elk_stack:    additionally deploy an ELK docker stack for event logging"
 -    echo -e "     -m <MODULE>:    install OSM but only rebuild the specified docker images (LW-UI, NBI, LCM, RO, MON, POL, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, NONE)"
 +    echo -e "     --pla:          install the PLA module for placement support"
 +    echo -e "     -m <MODULE>:    install OSM but only rebuild the specified docker images (LW-UI, NBI, LCM, RO, MON, POL, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, PLA, NONE)"
      echo -e "     -o <ADDON>:     ONLY (un)installs one of the addons (vimemu, elk_stack)"
      echo -e "     -D <devops path> use local devops installation path"
      echo -e "     -w <work dir>   Location to store runtime installation"
      echo -e "     -t <docker tag> specify osm docker tag (default is latest)"
 -    echo -e "     --nolxd:        do not install and configure LXD, allowing unattended installations (assumes LXD is already installed and confifured)"
 +    echo -e "     --nolxd:        do not install and configure LXD, allowing unattended installations (assumes LXD is already installed and configured)"
      echo -e "     --nodocker:     do not install docker, do not initialize a swarm (assumes docker is already installed and a swarm has been initialized)"
      echo -e "     --nojuju:       do not juju, assumes already installed"
      echo -e "     --nodockerbuild:do not build docker images (use existing locally cached images)"
      echo -e "     --showopts:     print chosen options and exit (only for debugging)"
      echo -e "     -y:             do not prompt for confirmation, assumes yes"
      echo -e "     -h / --help:    print this help"
 +    echo -e "     --charmed:                       install OSM with charms"
 +    echo -e "     --bundle <bundle path>:          Specify with which bundle to deploy OSM with charms (--charmed option)"
 +    echo -e "     --kubeconfig <kubeconfig path>:  Specify with which kubernetes to deploy OSM with charms (--charmed option)"
 +    echo -e "     --lxdendpoint <lxd endpoint ip>: Specify with which LXD to deploy OSM with charms (--charmed option)"
 +    echo -e "     --lxdcert <lxd cert path>:       Specify external LXD cert to deploy OSM with charms (--charmed option)"
 +    echo -e "     --microstack:                    Installs microstack as a vim. (--charmed option)"
 +
  }
  
  # takes a juju/accounts.yaml file and returns the password specific
@@@ -191,7 -183,6 +191,7 @@@ function uninstall_lightweight() 
          docker image rm ${DOCKER_USER}/nbi:${OSM_DOCKER_TAG}
          docker image rm ${DOCKER_USER}/mon:${OSM_DOCKER_TAG}
          docker image rm ${DOCKER_USER}/pol:${OSM_DOCKER_TAG}
 +        docker image rm ${DOCKER_USER}/pla:${OSM_DOCKER_TAG}
          docker image rm ${DOCKER_USER}/osmclient:${OSM_DOCKER_TAG}
  EONG
  
@@@ -321,7 -312,7 +321,7 @@@ function install_prometheus_nodeexporte
              sudo cp /tmp/node_exporter-$PROMETHEUS_NODE_EXPORTER_TAG.linux-amd64/node_exporter /usr/local/bin
              sudo chown node_exporter:node_exporter /usr/local/bin/node_exporter
              sudo rm -rf node_exporter-$PROMETHEUS_NODE_EXPORTER_TAG.linux-amd64*
-             sudo cp ${OSM_DEVOPS}/installers/docker/files/node_exporter.service /etc/systemd/system/node_exporter.service
+             sudo cp ${OSM_DEVOPS}/installers/docker/prometheus/node_exporter.service /etc/systemd/system/node_exporter.service
              sudo systemctl daemon-reload
              sudo systemctl restart node_exporter
              sudo systemctl enable node_exporter
@@@ -451,14 -442,6 +451,14 @@@ function generate_docker_images() 
          sg docker -c "docker build ${LWTEMPDIR}/POL -f ${LWTEMPDIR}/POL/docker/Dockerfile -t ${DOCKER_USER}/pol --no-cache" || FATAL "cannot build POL docker image"
      fi
  
 +    if [ -n "$PULL_IMAGES" -a -n "$INSTALL_PLA" ]; then
 +        sg docker -c "docker pull ${DOCKER_USER}/pla:${OSM_DOCKER_TAG}" || FATAL "cannot pull PLA docker image"
 +    elif [ -z "$TO_REBUILD" -a -n "$INSTALL_PLA" ] || echo $TO_REBUILD | grep -q PLA ; then
 +        git -C ${LWTEMPDIR} clone https://osm.etsi.org/gerrit/osm/PLA
 +        git -C ${LWTEMPDIR}/PLA checkout ${COMMIT_ID}
 +        sg docker -c "docker build ${LWTEMPDIR}/PLA -f ${LWTEMPDIR}/PLA/docker/Dockerfile -t ${DOCKER_USER}/pla --no-cache" || FATAL "cannot build PLA docker image"
 +    fi
 +
      if [ -n "$PULL_IMAGES" ]; then
          sg docker -c "docker pull ${DOCKER_USER}/nbi:${OSM_DOCKER_TAG}" || FATAL "cannot pull NBI docker image"
          sg docker -c "docker pull ${DOCKER_USER}/keystone:${OSM_DOCKER_TAG}" || FATAL "cannot pull KEYSTONE docker image"
@@@ -537,16 -520,21 +537,24 @@@ function generate_docker_env_files() 
      else
          # Docker-compose
          $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/docker-compose.yaml $OSM_DOCKER_WORK_DIR/docker-compose.yaml
 +        if [ -n "$INSTALL_PLA" ]; then
 +            $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/osm_pla/docker-compose.yaml $OSM_DOCKER_WORK_DIR/osm_pla/docker-compose.yaml
 +        fi
  
-         # Prometheus
-         $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/files/prometheus.yml $OSM_DOCKER_WORK_DIR/prometheus.yml
+         # Prometheus files
+         $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/prometheus
+         $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/prometheus/prometheus.yml $OSM_DOCKER_WORK_DIR/prometheus/prometheus.yml
  
-         # Grafana & Prometheus Exporter files
-         $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/files
-         $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/files/* $OSM_DOCKER_WORK_DIR/files/
+         # Grafana files
+         $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/grafana
+         $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/grafana/dashboards-osm.yml $OSM_DOCKER_WORK_DIR/grafana/dashboards-osm.yml
+         $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/grafana/datasource-prometheus.yml $OSM_DOCKER_WORK_DIR/grafana/datasource-prometheus.yml
+         $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/grafana/osm-sample-dashboard.json $OSM_DOCKER_WORK_DIR/grafana/osm-sample-dashboard.json
+         $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/grafana/osm-system-dashboard.json $OSM_DOCKER_WORK_DIR/grafana/osm-system-dashboard.json
+         # Prometheus Exporters files
+         $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/prometheus_exporters
+         $WORKDIR_SUDO cp -b ${OSM_DEVOPS}/installers/docker/prometheus_exporters/node_exporter.service $OSM_DOCKER_WORK_DIR/prometheus_exporters/node_exporter.service
      fi
  
      # LCM
@@@ -723,15 -711,6 +731,15 @@@ function deploy_osm_services() 
      kubectl apply -n $OSM_STACK_NAME -f $OSM_K8S_WORK_DIR
  }
  
 +function deploy_osm_pla_service() {
 +    # corresponding to parse_yaml
 +    [ ! $OSM_DOCKER_TAG == "7" ] && $WORKDIR_SUDO sed -i "s/opensourcemano\/pla:.*/opensourcemano\/pla:$OSM_DOCKER_TAG/g" $OSM_DOCKER_WORK_DIR/osm_pla/pla.yaml
 +    # corresponding to namespace_vol
 +    $WORKDIR_SUDO  sed -i "s#path: /var/lib/osm#path: $OSM_NAMESPACE_VOL#g" $OSM_DOCKER_WORK_DIR/osm_pla/pla.yaml
 +    # corresponding to deploy_osm_services
 +    kubectl apply -n $OSM_STACK_NAME -f $OSM_DOCKER_WORK_DIR/osm_pla
 +}
 +
  function parse_yaml() {
      osm_services="nbi lcm ro pol mon light-ui keystone"
      TAG=$1
@@@ -812,11 -791,7 +820,11 @@@ function deploy_lightweight() 
      echo "export GRAFANA_TAG=${GRAFANA_TAG}" | $WORKDIR_SUDO tee --append $OSM_DOCKER_WORK_DIR/osm_ports.sh
  
      pushd $OSM_DOCKER_WORK_DIR
 -    sg docker -c ". ./osm_ports.sh; docker stack deploy -c $OSM_DOCKER_WORK_DIR/docker-compose.yaml $OSM_STACK_NAME"
 +    if [ -n "$INSTALL_PLA" ]; then
 +        sg docker -c ". ./osm_ports.sh; docker stack deploy -c $OSM_DOCKER_WORK_DIR/docker-compose.yaml -c $OSM_DOCKER_WORK_DIR/osm_pla/docker-compose.yaml $OSM_STACK_NAME"
 +    else
 +        sg docker -c ". ./osm_ports.sh; docker stack deploy -c $OSM_DOCKER_WORK_DIR/docker-compose.yaml $OSM_STACK_NAME"
 +    fi
      popd
  
      echo "Finished deployment of lightweight build"
@@@ -876,7 -851,6 +884,7 @@@ function install_lightweight() 
      [ "${OSM_STACK_NAME}" == "osm" ] || OSM_DOCKER_WORK_DIR="$OSM_WORK_DIR/stack/$OSM_STACK_NAME"
      [ -n "$KUBERNETES" ] && OSM_K8S_WORK_DIR="$OSM_DOCKER_WORK_DIR/osm_pods" && OSM_NAMESPACE_VOL="${OSM_HOST_VOL}/${OSM_STACK_NAME}"
      [ ! -d "$OSM_DOCKER_WORK_DIR" ] && $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR
 +    [ ! -d "$OSM_DOCKER_WORK_DIR/osm_pla" -a -n "$INSTALL_PLA" ] && $WORKDIR_SUDO mkdir -p $OSM_DOCKER_WORK_DIR/osm_pla
      [ -n "$KUBERNETES" ] && $WORKDIR_SUDO cp -b $OSM_DEVOPS/installers/docker/cluster-config.yaml $OSM_DOCKER_WORK_DIR/cluster-config.yaml
  
      track checkingroot
          [ ! $OSM_DOCKER_TAG == "7" ] && parse_yaml $OSM_DOCKER_TAG
          namespace_vol
          deploy_osm_services
 +        if [ -n "$INSTALL_PLA"]; then
 +            # optional PLA install
 +            deploy_osm_pla_service
 +        fi
          track deploy_osm_services_k8s
          if [ -n "$INSTALL_K8S_MONITOR" ]; then
              # install OSM MONITORING
@@@ -1074,7 -1044,6 +1082,7 @@@ function dump_vars()
      echo "RECONFIGURE=$RECONFIGURE"
      echo "TEST_INSTALLER=$TEST_INSTALLER"
      echo "INSTALL_VIMEMU=$INSTALL_VIMEMU"
 +    echo "INSTALL_PLA=$INSTALL_PLA"
      echo "INSTALL_LXD=$INSTALL_LXD"
      echo "INSTALL_LIGHTWEIGHT=$INSTALL_LIGHTWEIGHT"
      echo "INSTALL_ONLY=$INSTALL_ONLY"
@@@ -1133,7 -1102,6 +1141,7 @@@ INSTALL_FROM_SOURCE="
  RELEASE="ReleaseSEVEN"
  REPOSITORY="stable"
  INSTALL_VIMEMU=""
 +INSTALL_PLA=""
  LXD_REPOSITORY_BASE="https://osm-download.etsi.org/repository/osm/lxd"
  LXD_REPOSITORY_PATH=""
  INSTALL_LIGHTWEIGHT="y"
@@@ -1229,7 -1197,6 +1237,7 @@@ while getopts ":b:r:c:k:u:R:D:o:m:H:S:s
              [ "${OPTARG}" == "KEYSTONE-DB" ] && TO_REBUILD="$TO_REBUILD KEYSTONE-DB" && continue
              [ "${OPTARG}" == "GRAFANA" ] && TO_REBUILD="$TO_REBUILD GRAFANA" && continue
              [ "${OPTARG}" == "NONE" ] && TO_REBUILD="$TO_REBUILD NONE" && continue
 +            [ "${OPTARG}" == "PLA" ] && TO_REBUILD="$TO_REBUILD PLA" && continue
              ;;
          H)
              OSM_VCA_HOST="${OPTARG}"
              [ "${OPTARG}" == "nohostclient" ] && INSTALL_NOHOSTCLIENT="y" && continue
              [ "${OPTARG}" == "pullimages" ] && continue
              [ "${OPTARG}" == "k8s_monitor" ] && INSTALL_K8S_MONITOR="y" && continue
 +            [ "${OPTARG}" == "charmed" ] && CHARMED="y" && continue
 +            [ "${OPTARG}" == "bundle" ] && continue
 +            [ "${OPTARG}" == "kubeconfig" ] && continue
 +            [ "${OPTARG}" == "lxdendpoint" ] && continue
 +            [ "${OPTARG}" == "lxdcert" ] && continue
 +            [ "${OPTARG}" == "microstack" ] && continue
 +            [ "${OPTARG}" == "pla" ] && INSTALL_PLA="y" && continue
              echo -e "Invalid option: '--$OPTARG'\n" >&2
              usage && exit 1
              ;;
  done
  
  [ -n "$TO_REBUILD" ] && [ "$TO_REBUILD" != " NONE" ] && echo $TO_REBUILD | grep -q NONE && FATAL "Incompatible option: -m NONE cannot be used with other -m options"
 +[ -n "$TO_REBUILD" ] && [ "$TO_REBUILD" == " PLA" ] && [ -z "$INSTALL_PLA" ] && FATAL "Incompatible option: -m PLA cannot be used without --pla option"
  
  if [ -n "$SHOWOPTS" ]; then
      dump_vars
      exit 0
  fi
  
 +if [ -n "$CHARMED" ]; then
 +     if [ -n "$UNINSTALL" ]; then
 +        /usr/share/osm-devops/installers/charmed_uninstall.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D /usr/share/osm-devops -t $DOCKER_TAG "$@"
 +     else
 +        /usr/share/osm-devops/installers/charmed_install.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D /usr/share/osm-devops -t $DOCKER_TAG "$@"
 +     fi
 +
 +     echo "Your installation is now complete, follow these steps for configuring the osmclient:"
 +     echo
 +     echo "1. Get the NBI IP with the following command:"
 +     echo
 +     echo "juju status --format yaml | yq r - applications.nbi-k8s.address"
 +     echo
 +     echo "2. Create the OSM_HOSTNAME environment variable with the NBI IP"
 +     echo
 +     echo "export OSM_HOSTNAME=<NBI-IP>"
 +     echo
 +     echo "3. Add the previous command to your .bashrc for other Shell sessions"
 +     echo
 +     echo "export OSM_HOSTNAME=<previous-IP> >> ~/.bashrc"
 +     echo
 +     echo "DONE"
 +
 +     exit 0
 +fi
 +
  # if develop, we force master
  [ -z "$COMMIT_ID" ] && [ -n "$DEVELOP" ] && COMMIT_ID="master"
  
@@@ -1415,4 -1348,3 +1423,4 @@@ export OSM_USE_LOCAL_DEVOPS=tru
  wget -q -O- https://osm-download.etsi.org/ftp/osm-7.0-seven/README2.txt &> /dev/null
  track end
  echo -e "\nDONE"
 +