osm_mon/collector/infra_collectors/base_osinfra.py

   1 # Copyright 2018 Whitestack, LLC
   2 # *************************************************************
   3
   4 # This file is part of OSM Monitoring module
   5 # All Rights Reserved to Whitestack, LLC
   6
   7 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   8 # not use this file except in compliance with the License. You may obtain
   9 # a copy of the License at
  10
  11 #         http://www.apache.org/licenses/LICENSE-2.0
  12
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  15 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  16 # License for the specific language governing permissions and limitations
  17 # under the License.
  18
  19 # For those usages not covered by the Apache License, Version 2.0 please
  20 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  21 ##
  22 import logging
  23 from typing import List
  24
  25 from keystoneclient.v3 import client as keystone_client
  26 from novaclient import client as nova_client
  27 from cinderclient import client as cinder_client
  28 from neutronclient.neutron import client as neutron_client
  29
  30 from osm_mon.collector.infra_collectors.base_vim import BaseVimInfraCollector
  31 from osm_mon.collector.metric import Metric
  32 from osm_mon.collector.utils.openstack import OpenstackUtils
  33 from osm_mon.core.common_db import CommonDbClient
  34 from osm_mon.core.config import Config
  35
  36 log = logging.getLogger(__name__)
  37
  38
  39 class BaseOpenStackInfraCollector(BaseVimInfraCollector):
  40     def __init__(self, config: Config, vim_account_id: str):
  41         super().__init__(config, vim_account_id)
  42         self.conf = config
  43         self.common_db = CommonDbClient(config)
  44         self.vim_account = self.common_db.get_vim_account(vim_account_id)
  45         # self.keystone = self._build_keystone_client(self.vim_account)
  46         self.vim_session = None
  47         self.nova = self._build_nova_client(self.vim_account)
  48         self.cinder = self._build_cinder_client(self.vim_account)
  49         self.neutron, self.tenant_id = self._build_neutron_client(self.vim_account)
  50
  51     def collect(self) -> List[Metric]:
  52         metrics = []
  53         vim_status = self.is_vim_ok()
  54         if vim_status:
  55             # Updating the resources in mongoDB
  56             self.update_resources()
  57         if self.vim_account["_admin"]["projects_read"]:
  58             vim_project_id = self.vim_account["_admin"]["projects_read"][0]
  59         else:
  60             vim_project_id = ""
  61         vim_tags = {
  62             "vim_account_id": self.vim_account["_id"],
  63             "project_id": vim_project_id,
  64         }
  65         vim_status_metric = Metric(vim_tags, "vim_status", vim_status)
  66         metrics.append(vim_status_metric)
  67         vnfrs = self.common_db.get_vnfrs(vim_account_id=self.vim_account["_id"])
  68         if self.conf.get("collector", "vm_infra_metrics"):
  69             vm_infra_metrics_enabled = str(
  70                 self.conf.get("collector", "vm_infra_metrics")
  71             ).lower() in ("yes", "true", "1")
  72         else:
  73             vm_infra_metrics_enabled = True
  74         if vm_infra_metrics_enabled:
  75             for vnfr in vnfrs:
  76                 nsr_id = vnfr["nsr-id-ref"]
  77                 ns_name = self.common_db.get_nsr(nsr_id)["name"]
  78                 vnf_member_index = vnfr["member-vnf-index-ref"]
  79                 if vnfr["_admin"]["projects_read"]:
  80                     vnfr_project_id = vnfr["_admin"]["projects_read"][0]
  81                 else:
  82                     vnfr_project_id = ""
  83                 for vdur in vnfr["vdur"]:
  84                     if "vim-id" not in vdur:
  85                         log.debug("Field vim-id is not present in vdur")
  86                         continue
  87                     resource_uuid = vdur["vim-id"]
  88                     tags = {
  89                         "vim_account_id": self.vim_account["_id"],
  90                         "resource_uuid": resource_uuid,
  91                         "ns_id": nsr_id,
  92                         "ns_name": ns_name,
  93                         "vnf_member_index": vnf_member_index,
  94                         "vdu_name": vdur.get("name", ""),
  95                         "project_id": vnfr_project_id,
  96                     }
  97                     try:
  98                         vm = self.nova.servers.get(resource_uuid)
  99                         vm_status = 0 if (vm.status == "ERROR") else 1
 100                         vm_status_metric = Metric(tags, "vm_status", vm_status)
 101                     except Exception as e:
 102                         log.warning("VM status is not OK: %s" % e)
 103                         vm_status_metric = Metric(tags, "vm_status", 0)
 104                     metrics.append(vm_status_metric)
 105
 106         return metrics
 107
 108     def is_vim_ok(self) -> bool:
 109         try:
 110             self.nova.servers.list()
 111             return True
 112         except Exception as e:
 113             log.warning("VIM status is not OK: %s" % e)
 114             return False
 115
 116     def update_resources(self):
 117         if "resources" in self.vim_account:
 118             vimacc_resources = self.vim_account["resources"]
 119             # Compute resources
 120             try:
 121                 com_lim = self.nova.limits.get()._info["absolute"]
 122                 if ("compute" in vimacc_resources) and (
 123                     (
 124                         vimacc_resources["compute"]["ram"]["total"]
 125                         != com_lim["maxTotalRAMSize"]
 126                     )
 127                     or (
 128                         vimacc_resources["compute"]["vcpus"]["total"]
 129                         != com_lim["maxTotalCores"]
 130                     )
 131                     or (
 132                         vimacc_resources["compute"]["ram"]["used"]
 133                         != com_lim["totalRAMUsed"]
 134                     )
 135                     or (
 136                         vimacc_resources["compute"]["vcpus"]["used"]
 137                         != com_lim["totalCoresUsed"]
 138                     )
 139                     or (
 140                         vimacc_resources["compute"]["instances"]["total"]
 141                         != com_lim["maxTotalInstances"]
 142                     )
 143                     or (
 144                         vimacc_resources["compute"]["instances"]["used"]
 145                         != com_lim["totalInstancesUsed"]
 146                     )
 147                 ):
 148                     update_dict = {
 149                         "resources.compute": {
 150                             "ram": {
 151                                 "total": com_lim["maxTotalRAMSize"],
 152                                 "used": com_lim["totalRAMUsed"],
 153                             },
 154                             "vcpus": {
 155                                 "total": com_lim["maxTotalCores"],
 156                                 "used": com_lim["totalCoresUsed"],
 157                             },
 158                             "instances": {
 159                                 "total": com_lim["maxTotalInstances"],
 160                                 "used": com_lim["totalInstancesUsed"],
 161                             },
 162                         }
 163                     }
 164                     suc_value = self.common_db.set_vim_account(
 165                         str(self.vim_account["_id"]), update_dict
 166                     )
 167                     log.info("Compute resources update in mongoDB  = %s" % suc_value)
 168             except Exception as e:
 169                 log.warning("Error in updating compute resources: %s" % e)
 170
 171             # Volume resources
 172             try:
 173                 vol_lim = self.cinder.limits.get()._info["absolute"]
 174                 if ("storage" in vimacc_resources) and (
 175                     (
 176                         vimacc_resources["storage"]["volumes"]["total"]
 177                         != vol_lim["maxTotalVolumes"]
 178                     )
 179                     or (
 180                         vimacc_resources["storage"]["snapshots"]["total"]
 181                         != vol_lim["maxTotalSnapshots"]
 182                     )
 183                     or (
 184                         vimacc_resources["storage"]["volumes"]["used"]
 185                         != vol_lim["totalVolumesUsed"]
 186                     )
 187                     or (
 188                         vimacc_resources["storage"]["snapshots"]["used"]
 189                         != vol_lim["totalSnapshotsUsed"]
 190                     )
 191                     or (
 192                         vimacc_resources["storage"]["storage"]["total"]
 193                         != vol_lim["maxTotalVolumeGigabytes"]
 194                     )
 195                     or (
 196                         vimacc_resources["storage"]["storage"]["used"]
 197                         != vol_lim["totalGigabytesUsed"]
 198                     )
 199                 ):
 200                     update_dict = {
 201                         "resources.storage": {
 202                             "volumes": {
 203                                 "total": vol_lim["maxTotalVolumes"],
 204                                 "used": vol_lim["totalVolumesUsed"],
 205                             },
 206                             "snapshots": {
 207                                 "total": vol_lim["maxTotalSnapshots"],
 208                                 "used": vol_lim["totalSnapshotsUsed"],
 209                             },
 210                             "storage": {
 211                                 "total": vol_lim["maxTotalVolumeGigabytes"],
 212                                 "used": vol_lim["totalGigabytesUsed"],
 213                             },
 214                         }
 215                     }
 216                     suc_value = self.common_db.set_vim_account(
 217                         str(self.vim_account["_id"]), update_dict
 218                     )
 219                     log.info("Volume resources update in mongoDB = %s" % suc_value)
 220             except Exception as e:
 221                 log.warning("Error in updating volume resources: %s" % e)
 222
 223             # Network resources
 224             try:
 225                 net_lim = self.neutron.show_quota_details(self.tenant_id)["quota"]
 226                 if ("network" in vimacc_resources) and (
 227                     (
 228                         vimacc_resources["network"]["networks"]["total"]
 229                         != net_lim["network"]["limit"]
 230                     )
 231                     or (
 232                         vimacc_resources["network"]["networks"]["used"]
 233                         != net_lim["network"]["used"]
 234                     )
 235                     or (
 236                         vimacc_resources["network"]["subnets"]["total"]
 237                         != net_lim["subnet"]["limit"]
 238                     )
 239                     or (
 240                         vimacc_resources["network"]["subnets"]["used"]
 241                         != net_lim["subnet"]["used"]
 242                     )
 243                     or (
 244                         vimacc_resources["network"]["floating_ips"]["total"]
 245                         != net_lim["floatingip"]["limit"]
 246                     )
 247                     or (
 248                         vimacc_resources["network"]["floating_ips"]["used"]
 249                         != net_lim["floatingip"]["used"]
 250                     )
 251                 ):
 252                     update_dict = {
 253                         "resources.network": {
 254                             "networks": {
 255                                 "total": net_lim["network"]["limit"],
 256                                 "used": net_lim["network"]["used"],
 257                             },
 258                             "subnets": {
 259                                 "total": net_lim["subnet"]["limit"],
 260                                 "used": net_lim["subnet"]["used"],
 261                             },
 262                             "floating_ips": {
 263                                 "total": net_lim["floatingip"]["limit"],
 264                                 "used": net_lim["floatingip"]["used"],
 265                             },
 266                         }
 267                     }
 268                     suc_value = self.common_db.set_vim_account(
 269                         str(self.vim_account["_id"]), update_dict
 270                     )
 271                     log.info("Network resources update in mongoDB = %s" % suc_value)
 272             except Exception as e:
 273                 log.warning("Error in updating network resources: %s" % e)
 274
 275     def _build_keystone_client(self, vim_account: dict) -> keystone_client.Client:
 276         sess = OpenstackUtils.get_session(vim_account)
 277         return keystone_client.Client(session=sess, timeout=10)
 278
 279     def _build_nova_client(self, vim_account: dict) -> nova_client.Client:
 280         sess = OpenstackUtils.get_session(vim_account)
 281         self.vim_session = sess
 282         return nova_client.Client("2", session=sess, timeout=10)
 283
 284     def _build_cinder_client(self, vim_account: dict) -> cinder_client.Client:
 285         # sess = OpenstackUtils.get_session(vim_account)
 286         return cinder_client.Client("3", session=self.vim_session, timeout=10)
 287
 288     def _build_neutron_client(self, vim_account: dict) -> tuple:
 289         # sess = OpenstackUtils.get_session(vim_account)
 290         tenant_id = self.vim_session.get_project_id()
 291         return (
 292             neutron_client.Client("2", session=self.vim_session, timeout=10),
 293             tenant_id,
 294         )