osm_mon/collector/vnf_collectors/vio.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2016-2017 VMware Inc.
   5 # This file is part of ETSI OSM
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact:  osslegalrouting@vmware.com
  22 ##
  23
  24 import json
  25 import logging
  26 import re
  27 import time
  28 import traceback
  29
  30 import requests
  31 import six
  32 from keystoneauth1 import session
  33 from keystoneauth1.identity import v3
  34 from novaclient import client as nClient
  35
  36 from osm_mon.collector.utils import CollectorUtils
  37 from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
  38 from osm_mon.collector.vnf_metric import VnfMetric
  39 from osm_mon.core.common_db import CommonDbClient
  40 from osm_mon.core.config import Config
  41 from osm_mon.core.database import VimCredentialsRepository, VimCredentials
  42
  43 log = logging.getLogger(__name__)
  44
  45 PERIOD_MSEC = {'HR': 3600000,
  46                'DAY': 86400000,
  47                'WEEK': 604800000,
  48                'MONTH': 2678400000,
  49                'YEAR': 31536000000}
  50
  51 METRIC_MAPPINGS = {
  52     "average_memory_utilization": "mem|usage_average",
  53     "cpu_utilization": "cpu|usage_average",
  54     "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average",
  55     "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average",
  56     "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average",
  57     "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average",
  58     "packets_dropped_0": "net:4000|dropped",
  59     "packets_dropped_1": "net:4001|dropped",
  60     "packets_dropped_2": "net:4002|dropped",
  61     "packets_received": "net:Aggregate of all instances|packetsRxPerSec",
  62     "packets_sent": "net:Aggregate of all instances|packetsTxPerSec",
  63 }
  64
  65
  66 class VIOCollector(BaseVimCollector):
  67     def __init__(self, config: Config, vim_account_id: str):
  68         super().__init__(config, vim_account_id)
  69         self.common_db = CommonDbClient(config)
  70         vim_account_info = CollectorUtils.get_credentials(vim_account_id)
  71         cfg = json.loads(vim_account_info.config)
  72         self.vrops_site = cfg['vrops_site']
  73         self.vrops_user = cfg['vrops_user']
  74         self.vrops_password = cfg['vrops_password']
  75         self.client = self.connect_client(vim_account_id)
  76
  77     def connect_client(self, vim_account_id: str):
  78         vim_account_details = VimCredentialsRepository.get(VimCredentials.uuid == vim_account_id)
  79         verify_ssl = CollectorUtils.is_verify_ssl(vim_account_details)
  80         auth = v3.Password(auth_url=vim_account_details.url,
  81                            username=vim_account_details.user,
  82                            password=vim_account_details.password,
  83                            project_name=vim_account_details.tenant_name,
  84                            project_domain_id='default',
  85                            user_domain_id='default')
  86         sess = session.Session(auth=auth, verify=verify_ssl)
  87         client = nClient.Client('2.1', session=sess, endpoint_type=None)
  88         return client
  89
  90     def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
  91         vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
  92         return vdur['vim-id']
  93
  94     def get_vm_name_and_id(self, resource_uuid):
  95         """ Method to find vm name and id using resource_uuid
  96         """
  97         server = self.client.servers.find(id=resource_uuid)
  98         name = server.to_dict()['name']
  99         _id = server.to_dict()['id']
 100         return name, _id
 101
 102     def get_vm_resource_id(self, name, _id):
 103         """ Find resource ID in vROPs
 104         """
 105         api_url = '/suite-api/api/resources?resourceKind=VirtualMachine'
 106         headers = {'Accept': 'application/json'}
 107
 108         resp = requests.get(self.vrops_site + api_url,
 109                             auth=(self.vrops_user, self.vrops_password),
 110                             verify=False, headers=headers)
 111
 112         if resp.status_code != 200:
 113             log.error("Failed to get resource details for{} {} {}".format(name,
 114                                                                           resp.status_code,
 115                                                                           resp.content))
 116             return None
 117
 118         vm_resource_id = None
 119         try:
 120             resp_data = json.loads(resp.content.decode('utf-8'))
 121             if resp_data.get('resourceList') is not None:
 122                 resource_list = resp_data.get('resourceList')
 123                 for resource in resource_list:
 124                     if resource.get('resourceKey') is not None:
 125                         m = re.match(r"(.*?)\s\((.*?)\)", resource['resourceKey']['name'])
 126                         if m:
 127                             v_name = m.group(1)
 128                             v_id = m.group(2)
 129                         if name == v_name and _id == v_id:
 130                             vm_resource_id = resource['identifier']
 131                             log.info("Found VM resource ID: {} for vm: {}".format(vm_resource_id,
 132                                                                                   v_name))
 133
 134         except Exception as exp:
 135             log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc()))
 136
 137         return vm_resource_id
 138
 139     def collect(self, vnfr: dict):
 140         nsr_id = vnfr['nsr-id-ref']
 141         vnf_member_index = vnfr['member-vnf-index-ref']
 142         vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
 143         metrics = []
 144         for vdur in vnfr['vdur']:
 145             # This avoids errors when vdur records have not been completely filled
 146             if 'name' not in vdur:
 147                 continue
 148             vdu = next(
 149                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
 150
 151             )
 152             if 'monitoring-param' in vdu:
 153                 for param in vdu['monitoring-param']:
 154                     metric_name = param['nfvi-metric']
 155                     vrops_metric_name = METRIC_MAPPINGS[metric_name]
 156                     resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
 157
 158                     name, _id = self.get_vm_name_and_id(resource_uuid)
 159                     if name and _id is not None:
 160                         resource_id = self.get_vm_resource_id(name, _id)
 161                     else:
 162                         return
 163                     try:
 164                         end_time = int(round(time.time() * 1000))
 165                         time_diff = PERIOD_MSEC['YEAR']
 166                         begin_time = end_time - time_diff
 167
 168                         api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format(
 169                             resource_id, vrops_metric_name, str(begin_time), str(end_time))
 170
 171                         headers = {'Accept': 'application/json'}
 172
 173                         resp = requests.get(self.vrops_site + api_url,
 174                                             auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
 175                                             )
 176
 177                         if resp.status_code != 200:
 178                             log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name,
 179                                                                                                  resp.status_code,
 180                                                                                                  resp.content))
 181                             return
 182
 183                         metrics_data = {}
 184                         m_data = json.loads(resp.content.decode('utf-8'))
 185
 186                         for resp_key, resp_val in six.iteritems(m_data):
 187                             if resp_key == 'values':
 188                                 data = m_data['values'][0]
 189                                 for data_k, data_v in six.iteritems(data):
 190                                     if data_k == 'stat-list':
 191                                         stat_list = data_v
 192                                         for stat_list_k, stat_list_v in six.iteritems(stat_list):
 193                                             for stat_keys, stat_vals in six.iteritems(stat_list_v[0]):
 194                                                 if stat_keys == 'timestamps':
 195                                                     metrics_data['time_series'] = stat_list_v[0]['timestamps']
 196                                                 if stat_keys == 'data':
 197                                                     metrics_data['metrics_series'] = stat_list_v[0]['data']
 198
 199                         if metrics_data:
 200                             metric = VnfMetric(nsr_id,
 201                                                vnf_member_index,
 202                                                vdur['name'],
 203                                                metric_name,
 204                                                metrics_data['metrics_series'][-1])
 205
 206                             metrics.append(metric)
 207
 208                     except Exception as e:
 209                         log.debug("No metric found: %s", e)
 210                         pass
 211
 212         return metrics