osm_mon/collector/vnf_collectors/vmware.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2016-2017 VMware Inc.
   5 # This file is part of ETSI OSM
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact:  osslegalrouting@vmware.com
  22 ##
  23
  24 import json
  25 import logging
  26 import time
  27 import traceback
  28 from xml.etree import ElementTree as XmlElementTree
  29
  30 import requests
  31 import six
  32 from pyvcloud.vcd.client import BasicLoginCredentials
  33 from pyvcloud.vcd.client import Client
  34
  35 from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
  36 from osm_mon.collector.vnf_metric import VnfMetric
  37 from osm_mon.core.auth import AuthManager
  38 from osm_mon.core.common_db import CommonDbClient
  39 from osm_mon.core.settings import Config
  40
  41 log = logging.getLogger(__name__)
  42
  43 API_VERSION = '5.9'
  44
  45 PERIOD_MSEC = {'HR': 3600000,
  46                'DAY': 86400000,
  47                'WEEK': 604800000,
  48                'MONTH': 2678400000,
  49                'YEAR': 31536000000}
  50
  51 METRIC_MAPPINGS = {
  52     "average_memory_utilization": "mem|usage_average",
  53     "cpu_utilization": "cpu|usage_average",
  54     "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average",
  55     "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average",
  56     "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average",
  57     "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average",
  58     "packets_dropped_0": "net:4000|dropped",
  59     "packets_dropped_1": "net:4001|dropped",
  60     "packets_dropped_2": "net:4002|dropped",
  61     "packets_received": "net:Aggregate of all instances|packetsRxPerSec",
  62     "packets_sent": "net:Aggregate of all instances|packetsTxPerSec",
  63 }
  64
  65 # Disable warnings from self-signed certificates.
  66 requests.packages.urllib3.disable_warnings()
  67
  68
  69 class VMwareCollector(BaseVimCollector):
  70     def __init__(self, vim_account_id: str):
  71         super().__init__(vim_account_id)
  72         self.common_db = CommonDbClient()
  73         self.auth_manager = AuthManager()
  74         self.granularity = self._get_granularity(vim_account_id)
  75         vim_account = self.get_vim_account(vim_account_id)
  76         self.vrops_site = vim_account['vrops_site']
  77         self.vrops_user = vim_account['vrops_user']
  78         self.vrops_password = vim_account['vrops_password']
  79         self.vcloud_site = vim_account['vim_url']
  80         self.admin_username = vim_account['admin_username']
  81         self.admin_password = vim_account['admin_password']
  82         self.vim_uuid = vim_account['vim_uuid']
  83
  84     def connect_as_admin(self):
  85         """ Method connect as pvdc admin user to vCloud director.
  86             There are certain action that can be done only by provider vdc admin user.
  87             Organization creation / provider network creation etc.
  88
  89             Returns:
  90                 The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
  91         """
  92
  93         log.info("Logging into vCD org as admin.")
  94
  95         try:
  96             host = self.vcloud_site
  97             admin_user = self.admin_username
  98             admin_passwd = self.admin_password
  99             org = 'System'
 100             client = Client(host, verify_ssl_certs=False)
 101             client.set_credentials(BasicLoginCredentials(admin_user, org,
 102                                                          admin_passwd))
 103             return client
 104
 105         except Exception as e:
 106             log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
 107
 108     def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
 109         vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
 110         return vdur['vim-id']
 111
 112     def get_vim_account(self, vim_account_id: str):
 113         """
 114            Method to get VIM account details by its ID
 115            arg - VIM ID
 116            return - dict with vim account details
 117         """
 118         vim_account = {}
 119         vim_account_info = self.auth_manager.get_credentials(vim_account_id)
 120
 121         vim_account['name'] = vim_account_info.name
 122         vim_account['vim_tenant_name'] = vim_account_info.tenant_name
 123         vim_account['vim_type'] = vim_account_info.type
 124         vim_account['vim_url'] = vim_account_info.url
 125         vim_account['org_user'] = vim_account_info.user
 126         vim_account['org_password'] = vim_account_info.password
 127         vim_account['vim_uuid'] = vim_account_info.uuid
 128
 129         vim_config = json.loads(vim_account_info.config)
 130         vim_account['admin_username'] = vim_config['admin_username']
 131         vim_account['admin_password'] = vim_config['admin_password']
 132         vim_account['vrops_site'] = vim_config['vrops_site']
 133         vim_account['vrops_user'] = vim_config['vrops_user']
 134         vim_account['vrops_password'] = vim_config['vrops_password']
 135         vim_account['vcenter_ip'] = vim_config['vcenter_ip']
 136         vim_account['vcenter_port'] = vim_config['vcenter_port']
 137         vim_account['vcenter_user'] = vim_config['vcenter_user']
 138         vim_account['vcenter_password'] = vim_config['vcenter_password']
 139
 140         if vim_config['nsx_manager'] is not None:
 141             vim_account['nsx_manager'] = vim_config['nsx_manager']
 142
 143         if vim_config['nsx_user'] is not None:
 144             vim_account['nsx_user'] = vim_config['nsx_user']
 145
 146         if vim_config['nsx_password'] is not None:
 147             vim_account['nsx_password'] = vim_config['nsx_password']
 148
 149         if vim_config['orgname'] is not None:
 150             vim_account['orgname'] = vim_config['orgname']
 151
 152         return vim_account
 153
 154     def _get_granularity(self, vim_account_id: str):
 155         creds = self.auth_manager.get_credentials(vim_account_id)
 156         vim_config = json.loads(creds.config)
 157         if 'granularity' in vim_config:
 158             return int(vim_config['granularity'])
 159         else:
 160             cfg = Config.instance()
 161             return cfg.OS_DEFAULT_GRANULARITY
 162
 163     def get_vm_moref_id(self, vapp_uuid):
 164         """
 165            Method to get the moref_id of given VM
 166            arg - vapp_uuid
 167            return - VM mored_id
 168         """
 169         try:
 170             if vapp_uuid:
 171                 vm_details = self.get_vapp_details_rest(vapp_uuid)
 172
 173                 if vm_details and "vm_vcenter_info" in vm_details:
 174                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
 175
 176             log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
 177             return vm_moref_id
 178
 179         except Exception as exp:
 180             log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc()))
 181
 182     def get_vapp_details_rest(self, vapp_uuid=None):
 183         """
 184         Method retrieve vapp detail from vCloud director
 185         vapp_uuid - is vapp identifier.
 186         Returns - VM MOref ID or return None
 187         """
 188         parsed_respond = {}
 189         vca = None
 190
 191         if vapp_uuid is None:
 192             return parsed_respond
 193
 194         vca = self.connect_as_admin()
 195
 196         if not vca:
 197             log.error("Failed to connect to vCD")
 198             return parsed_respond
 199
 200         url_list = [self.vcloud_site, '/api/vApp/vapp-', vapp_uuid]
 201         get_vapp_restcall = ''.join(url_list)
 202
 203         if vca._session:
 204             headers = {'Accept': 'application/*+xml;version=' + API_VERSION,
 205                        'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
 206             response = requests.get(get_vapp_restcall,
 207                                     headers=headers,
 208                                     verify=False)
 209
 210             if response.status_code != 200:
 211                 log.error("REST API call {} failed. Return status code {}".format(get_vapp_restcall,
 212                                                                                   response.content))
 213                 return parsed_respond
 214
 215             try:
 216                 xmlroot_respond = XmlElementTree.fromstring(response.content)
 217
 218                 namespaces = {'vm': 'http://www.vmware.com/vcloud/v1.5',
 219                               "vmext": "http://www.vmware.com/vcloud/extension/v1.5",
 220                               "xmlns": "http://www.vmware.com/vcloud/v1.5"}
 221
 222                 # parse children section for other attrib
 223                 children_section = xmlroot_respond.find('vm:Children/', namespaces)
 224                 if children_section is not None:
 225                     vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
 226                     if vCloud_extension_section is not None:
 227                         vm_vcenter_info = {}
 228                         vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
 229                         vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
 230                         if vmext is not None:
 231                             vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
 232                         parsed_respond["vm_vcenter_info"] = vm_vcenter_info
 233
 234             except Exception as exp:
 235                 log.info("Error occurred for getting vApp details: {}\n{}".format(exp,
 236                                                                                   traceback.format_exc())
 237                          )
 238
 239         return parsed_respond
 240
 241     def get_vm_resource_id(self, vm_moref_id):
 242         """ Find resource ID in vROPs using vm_moref_id
 243         """
 244         if vm_moref_id is None:
 245             return None
 246
 247         api_url = '/suite-api/api/resources?resourceKind=VirtualMachine'
 248         headers = {'Accept': 'application/json'}
 249
 250         resp = requests.get(self.vrops_site + api_url,
 251                             auth=(self.vrops_user, self.vrops_password),
 252                             verify=False, headers=headers)
 253
 254         if resp.status_code != 200:
 255             log.error("Failed to get resource details for{} {} {}".format(vm_moref_id,
 256                                                                           resp.status_code,
 257                                                                           resp.content))
 258             return None
 259
 260         vm_resource_id = None
 261         try:
 262             resp_data = json.loads(resp.content.decode('utf-8'))
 263             if resp_data.get('resourceList') is not None:
 264                 resource_list = resp_data.get('resourceList')
 265                 for resource in resource_list:
 266                     if resource.get('resourceKey') is not None:
 267                         resource_details = resource['resourceKey']
 268                         if resource_details.get('resourceIdentifiers') is not None:
 269                             resource_identifiers = resource_details['resourceIdentifiers']
 270                             for resource_identifier in resource_identifiers:
 271                                 if resource_identifier['identifierType']['name'] == 'VMEntityObjectID':
 272                                     if resource_identifier.get('value') is not None and \
 273                                             resource_identifier['value'] == vm_moref_id:
 274                                         vm_resource_id = resource['identifier']
 275                                         log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id,
 276                                                                                                        vm_moref_id))
 277
 278         except Exception as exp:
 279             log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc()))
 280
 281         return vm_resource_id
 282
 283     def collect(self, vnfr: dict):
 284         nsr_id = vnfr['nsr-id-ref']
 285         vnf_member_index = vnfr['member-vnf-index-ref']
 286         vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
 287         metrics = []
 288         for vdur in vnfr['vdur']:
 289             # This avoids errors when vdur records have not been completely filled
 290             if 'name' not in vdur:
 291                 continue
 292             vdu = next(
 293                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
 294
 295             )
 296             if 'monitoring-param' in vdu:
 297                 for param in vdu['monitoring-param']:
 298                     metric_name = param['nfvi-metric']
 299                     vrops_metric_name = METRIC_MAPPINGS[metric_name]
 300                     resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
 301
 302                     # Find vm_moref_id from vApp uuid in vCD
 303                     vm_moref_id = self.get_vm_moref_id(resource_uuid)
 304                     if vm_moref_id is None:
 305                         log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid))
 306                         return
 307
 308                     # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
 309                     resource_id = self.get_vm_resource_id(vm_moref_id)
 310                     if resource_id is None:
 311                         log.debug("Failed to find resource in vROPs: {}".format(resource_uuid))
 312                         return
 313                     try:
 314                         end_time = int(round(time.time() * 1000))
 315                         time_diff = PERIOD_MSEC['YEAR']
 316                         begin_time = end_time - time_diff
 317
 318                         api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format(
 319                             resource_id, vrops_metric_name, str(begin_time), str(end_time))
 320
 321                         headers = {'Accept': 'application/json'}
 322
 323                         resp = requests.get(self.vrops_site + api_url,
 324                                             auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
 325                                             )
 326
 327                         if resp.status_code != 200:
 328                             log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name,
 329                                                                                                  resp.status_code,
 330                                                                                                  resp.content))
 331                             return
 332
 333                         metrics_data = {}
 334                         m_data = json.loads(resp.content.decode('utf-8'))
 335
 336                         for resp_key, resp_val in six.iteritems(m_data):
 337                             if resp_key == 'values':
 338                                 data = m_data['values'][0]
 339                                 for data_k, data_v in six.iteritems(data):
 340                                     if data_k == 'stat-list':
 341                                         stat_list = data_v
 342                                         for stat_list_k, stat_list_v in six.iteritems(stat_list):
 343                                             for stat_keys, stat_vals in six.iteritems(stat_list_v[0]):
 344                                                 if stat_keys == 'timestamps':
 345                                                     metrics_data['time_series'] = stat_list_v[0]['timestamps']
 346                                                 if stat_keys == 'data':
 347                                                     metrics_data['metrics_series'] = stat_list_v[0]['data']
 348
 349                         if metrics_data:
 350                             metric = VnfMetric(nsr_id,
 351                                                vnf_member_index,
 352                                                vdur['name'],
 353                                                metric_name,
 354                                                metrics_data['metrics_series'][-1])
 355
 356                             metrics.append(metric)
 357
 358                     except Exception as e:
 359                         log.debug("No metric found: %s", e)
 360                         pass
 361         return metrics