osm_mon/collector/vnf_collectors/vmware.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2016-2019 VMware Inc.
   5 # This file is part of ETSI OSM
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact:  osslegalrouting@vmware.com
  22 ##
  23
  24 import json
  25 import logging
  26 import time
  27 import traceback
  28 from xml.etree import ElementTree as XmlElementTree
  29
  30 import requests
  31 import six
  32 from pyvcloud.vcd.client import BasicLoginCredentials
  33 from pyvcloud.vcd.client import Client
  34
  35 from osm_mon.collector.utils.collector import CollectorUtils
  36 from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
  37 from osm_mon.collector.vnf_metric import VnfMetric
  38 from osm_mon.core.common_db import CommonDbClient
  39 from osm_mon.core.config import Config
  40
  41 log = logging.getLogger(__name__)
  42
  43 API_VERSION = '27.0'
  44
  45 PERIOD_MSEC = {'HR': 3600000,
  46                'DAY': 86400000,
  47                'WEEK': 604800000,
  48                'MONTH': 2678400000,
  49                'YEAR': 31536000000}
  50
  51 METRIC_MAPPINGS = {
  52     "average_memory_utilization": "mem|usage_average",
  53     "cpu_utilization": "cpu|usage_average",
  54     "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average",
  55     "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average",
  56     "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average",
  57     "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average",
  58     "packets_dropped_0": "net:4000|dropped",
  59     "packets_dropped_1": "net:4001|dropped",
  60     "packets_dropped_2": "net:4002|dropped",
  61     "packets_received": "net:Aggregate of all instances|packetsRxPerSec",
  62     "packets_sent": "net:Aggregate of all instances|packetsTxPerSec",
  63 }
  64
  65
  66 class VMwareCollector(BaseVimCollector):
  67     def __init__(self, config: Config, vim_account_id: str):
  68         super().__init__(config, vim_account_id)
  69         self.common_db = CommonDbClient(config)
  70         vim_account = self.get_vim_account(vim_account_id)
  71         self.vrops_site = vim_account['vrops_site']
  72         self.vrops_user = vim_account['vrops_user']
  73         self.vrops_password = vim_account['vrops_password']
  74         self.vcloud_site = vim_account['vim_url']
  75         self.admin_username = vim_account['admin_username']
  76         self.admin_password = vim_account['admin_password']
  77         self.vim_uuid = vim_account['vim_uuid']
  78
  79     def connect_as_admin(self):
  80         """ Method connect as pvdc admin user to vCloud director.
  81             There are certain action that can be done only by provider vdc admin user.
  82             Organization creation / provider network creation etc.
  83
  84             Returns:
  85                 The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
  86         """
  87
  88         log.info("Logging into vCD org as admin.")
  89
  90         admin_user = None
  91         try:
  92             host = self.vcloud_site
  93             admin_user = self.admin_username
  94             admin_passwd = self.admin_password
  95             org = 'System'
  96             client = Client(host, verify_ssl_certs=False)
  97             client.set_highest_supported_version()
  98             client.set_credentials(BasicLoginCredentials(admin_user, org,
  99                                                          admin_passwd))
 100             return client
 101
 102         except Exception as e:
 103             log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
 104
 105     def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
 106         vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
 107         return vdur['vim-id']
 108
 109     def get_vim_account(self, vim_account_id: str):
 110         """
 111            Method to get VIM account details by its ID
 112            arg - VIM ID
 113            return - dict with vim account details
 114         """
 115         vim_account = {}
 116         vim_account_info = CollectorUtils.get_credentials(vim_account_id)
 117
 118         vim_account['name'] = vim_account_info.name
 119         vim_account['vim_tenant_name'] = vim_account_info.tenant_name
 120         vim_account['vim_type'] = vim_account_info.type
 121         vim_account['vim_url'] = vim_account_info.url
 122         vim_account['org_user'] = vim_account_info.user
 123         vim_account['org_password'] = vim_account_info.password
 124         vim_account['vim_uuid'] = vim_account_info.uuid
 125
 126         vim_config = json.loads(vim_account_info.config)
 127         vim_account['admin_username'] = vim_config['admin_username']
 128         vim_account['admin_password'] = vim_config['admin_password']
 129         vim_account['vrops_site'] = vim_config['vrops_site']
 130         vim_account['vrops_user'] = vim_config['vrops_user']
 131         vim_account['vrops_password'] = vim_config['vrops_password']
 132         vim_account['vcenter_ip'] = vim_config['vcenter_ip']
 133         vim_account['vcenter_port'] = vim_config['vcenter_port']
 134         vim_account['vcenter_user'] = vim_config['vcenter_user']
 135         vim_account['vcenter_password'] = vim_config['vcenter_password']
 136
 137         if vim_config['nsx_manager'] is not None:
 138             vim_account['nsx_manager'] = vim_config['nsx_manager']
 139
 140         if vim_config['nsx_user'] is not None:
 141             vim_account['nsx_user'] = vim_config['nsx_user']
 142
 143         if vim_config['nsx_password'] is not None:
 144             vim_account['nsx_password'] = vim_config['nsx_password']
 145
 146         if vim_config['orgname'] is not None:
 147             vim_account['orgname'] = vim_config['orgname']
 148
 149         return vim_account
 150
 151     def get_vm_moref_id(self, vapp_uuid):
 152         """
 153            Method to get the moref_id of given VM
 154            arg - vapp_uuid
 155            return - VM mored_id
 156         """
 157         vm_moref_id = None
 158         try:
 159             if vapp_uuid:
 160                 vm_details = self.get_vapp_details_rest(vapp_uuid)
 161
 162                 if vm_details and "vm_vcenter_info" in vm_details:
 163                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
 164
 165             log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
 166             return vm_moref_id
 167
 168         except Exception as exp:
 169             log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc()))
 170
 171     def get_vapp_details_rest(self, vapp_uuid=None):
 172         """
 173         Method retrieve vapp detail from vCloud director
 174         vapp_uuid - is vapp identifier.
 175         Returns - VM MOref ID or return None
 176         """
 177         parsed_respond = {}
 178         vca = None
 179
 180         if vapp_uuid is None:
 181             return parsed_respond
 182
 183         vca = self.connect_as_admin()
 184
 185         if not vca:
 186             log.error("Failed to connect to vCD")
 187             return parsed_respond
 188
 189         url_list = [self.vcloud_site, '/api/vApp/vapp-', vapp_uuid]
 190         get_vapp_restcall = ''.join(url_list)
 191
 192         if vca._session:
 193             headers = {'Accept': 'application/*+xml;version=' + API_VERSION,
 194                        'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
 195             response = requests.get(get_vapp_restcall,
 196                                     headers=headers,
 197                                     verify=False)
 198
 199             if response.status_code != 200:
 200                 log.error("REST API call {} failed. Return status code {}".format(get_vapp_restcall,
 201                                                                                   response.content))
 202                 return parsed_respond
 203
 204             try:
 205                 xmlroot_respond = XmlElementTree.fromstring(response.content)
 206
 207                 namespaces = {'vm': 'http://www.vmware.com/vcloud/v1.5',
 208                               "vmext": "http://www.vmware.com/vcloud/extension/v1.5",
 209                               "xmlns": "http://www.vmware.com/vcloud/v1.5"}
 210
 211                 # parse children section for other attrib
 212                 children_section = xmlroot_respond.find('vm:Children/', namespaces)
 213                 if children_section is not None:
 214                     vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
 215                     if vCloud_extension_section is not None:
 216                         vm_vcenter_info = {}
 217                         vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
 218                         vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
 219                         if vmext is not None:
 220                             vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
 221                         parsed_respond["vm_vcenter_info"] = vm_vcenter_info
 222
 223             except Exception as exp:
 224                 log.info("Error occurred for getting vApp details: {}\n{}".format(exp,
 225                                                                                   traceback.format_exc())
 226                          )
 227
 228         return parsed_respond
 229
 230     def get_vm_resource_id(self, vm_moref_id):
 231         """ Find resource ID in vROPs using vm_moref_id
 232         """
 233         if vm_moref_id is None:
 234             return None
 235
 236         api_url = '/suite-api/api/resources?resourceKind=VirtualMachine'
 237         headers = {'Accept': 'application/json'}
 238
 239         resp = requests.get(self.vrops_site + api_url,
 240                             auth=(self.vrops_user, self.vrops_password),
 241                             verify=False, headers=headers)
 242
 243         if resp.status_code != 200:
 244             log.error("Failed to get resource details for{} {} {}".format(vm_moref_id,
 245                                                                           resp.status_code,
 246                                                                           resp.content))
 247             return None
 248
 249         vm_resource_id = None
 250         try:
 251             resp_data = json.loads(resp.content.decode('utf-8'))
 252             if resp_data.get('resourceList') is not None:
 253                 resource_list = resp_data.get('resourceList')
 254                 for resource in resource_list:
 255                     if resource.get('resourceKey') is not None:
 256                         resource_details = resource['resourceKey']
 257                         if resource_details.get('resourceIdentifiers') is not None:
 258                             resource_identifiers = resource_details['resourceIdentifiers']
 259                             for resource_identifier in resource_identifiers:
 260                                 if resource_identifier['identifierType']['name'] == 'VMEntityObjectID':
 261                                     if resource_identifier.get('value') is not None and \
 262                                             resource_identifier['value'] == vm_moref_id:
 263                                         vm_resource_id = resource['identifier']
 264                                         log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id,
 265                                                                                                        vm_moref_id))
 266
 267         except Exception as exp:
 268             log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc()))
 269
 270         return vm_resource_id
 271
 272     def collect(self, vnfr: dict):
 273         nsr_id = vnfr['nsr-id-ref']
 274         vnf_member_index = vnfr['member-vnf-index-ref']
 275         vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
 276         metrics = []
 277         for vdur in vnfr['vdur']:
 278             # This avoids errors when vdur records have not been completely filled
 279             if 'name' not in vdur:
 280                 continue
 281             vdu = next(
 282                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
 283
 284             )
 285             if 'monitoring-param' in vdu:
 286                 for param in vdu['monitoring-param']:
 287                     metric_name = param['nfvi-metric']
 288                     vrops_metric_name = METRIC_MAPPINGS[metric_name]
 289                     resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
 290
 291                     # Find vm_moref_id from vApp uuid in vCD
 292                     vm_moref_id = self.get_vm_moref_id(resource_uuid)
 293                     if vm_moref_id is None:
 294                         log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid))
 295                         return
 296
 297                     # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
 298                     resource_id = self.get_vm_resource_id(vm_moref_id)
 299                     if resource_id is None:
 300                         log.debug("Failed to find resource in vROPs: {}".format(resource_uuid))
 301                         return
 302                     try:
 303                         end_time = int(round(time.time() * 1000))
 304                         time_diff = PERIOD_MSEC['YEAR']
 305                         begin_time = end_time - time_diff
 306
 307                         api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format(
 308                             resource_id, vrops_metric_name, str(begin_time), str(end_time))
 309
 310                         headers = {'Accept': 'application/json'}
 311
 312                         resp = requests.get(self.vrops_site + api_url,
 313                                             auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
 314                                             )
 315
 316                         if resp.status_code != 200:
 317                             log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name,
 318                                                                                                  resp.status_code,
 319                                                                                                  resp.content))
 320                             return
 321
 322                         metrics_data = {}
 323                         m_data = json.loads(resp.content.decode('utf-8'))
 324
 325                         for resp_key, resp_val in six.iteritems(m_data):
 326                             if resp_key == 'values':
 327                                 data = m_data['values'][0]
 328                                 for data_k, data_v in six.iteritems(data):
 329                                     if data_k == 'stat-list':
 330                                         stat_list = data_v
 331                                         for stat_list_k, stat_list_v in six.iteritems(stat_list):
 332                                             for stat_keys, stat_vals in six.iteritems(stat_list_v[0]):
 333                                                 if stat_keys == 'timestamps':
 334                                                     metrics_data['time_series'] = stat_list_v[0]['timestamps']
 335                                                 if stat_keys == 'data':
 336                                                     metrics_data['metrics_series'] = stat_list_v[0]['data']
 337
 338                         if metrics_data:
 339                             metric = VnfMetric(nsr_id,
 340                                                vnf_member_index,
 341                                                vdur['name'],
 342                                                metric_name,
 343                                                metrics_data['metrics_series'][-1])
 344
 345                             metrics.append(metric)
 346
 347                     except Exception as e:
 348                         log.debug("No metric found: %s", e)
 349                         pass
 350         return metrics