osm_mon/collector/vnf_collectors/vmware.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2016-2017 VMware Inc.
   5 # This file is part of ETSI OSM
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact:  osslegalrouting@vmware.com
  22 ##
  23
  24 import json
  25 import logging
  26 import time
  27 import traceback
  28 from xml.etree import ElementTree as XmlElementTree
  29
  30 import requests
  31 import six
  32 from pyvcloud.vcd.client import BasicLoginCredentials
  33 from pyvcloud.vcd.client import Client
  34
  35 from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
  36 from osm_mon.collector.vnf_metric import VnfMetric
  37 from osm_mon.core.auth import AuthManager
  38 from osm_mon.core.common_db import CommonDbClient
  39 from osm_mon.core.config import Config
  40
  41 log = logging.getLogger(__name__)
  42
  43 API_VERSION = '5.9'
  44
  45 PERIOD_MSEC = {'HR': 3600000,
  46                'DAY': 86400000,
  47                'WEEK': 604800000,
  48                'MONTH': 2678400000,
  49                'YEAR': 31536000000}
  50
  51 METRIC_MAPPINGS = {
  52     "average_memory_utilization": "mem|usage_average",
  53     "cpu_utilization": "cpu|usage_average",
  54     "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average",
  55     "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average",
  56     "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average",
  57     "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average",
  58     "packets_dropped_0": "net:4000|dropped",
  59     "packets_dropped_1": "net:4001|dropped",
  60     "packets_dropped_2": "net:4002|dropped",
  61     "packets_received": "net:Aggregate of all instances|packetsRxPerSec",
  62     "packets_sent": "net:Aggregate of all instances|packetsTxPerSec",
  63 }
  64
  65 # Disable warnings from self-signed certificates.
  66 requests.packages.urllib3.disable_warnings()
  67
  68
  69 class VMwareCollector(BaseVimCollector):
  70     def __init__(self, config: Config, vim_account_id: str):
  71         super().__init__(config, vim_account_id)
  72         self.common_db = CommonDbClient(config)
  73         self.auth_manager = AuthManager(config)
  74         vim_account = self.get_vim_account(vim_account_id)
  75         self.vrops_site = vim_account['vrops_site']
  76         self.vrops_user = vim_account['vrops_user']
  77         self.vrops_password = vim_account['vrops_password']
  78         self.vcloud_site = vim_account['vim_url']
  79         self.admin_username = vim_account['admin_username']
  80         self.admin_password = vim_account['admin_password']
  81         self.vim_uuid = vim_account['vim_uuid']
  82
  83     def connect_as_admin(self):
  84         """ Method connect as pvdc admin user to vCloud director.
  85             There are certain action that can be done only by provider vdc admin user.
  86             Organization creation / provider network creation etc.
  87
  88             Returns:
  89                 The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
  90         """
  91
  92         log.info("Logging into vCD org as admin.")
  93
  94         admin_user = None
  95         try:
  96             host = self.vcloud_site
  97             admin_user = self.admin_username
  98             admin_passwd = self.admin_password
  99             org = 'System'
 100             client = Client(host, verify_ssl_certs=False)
 101             client.set_credentials(BasicLoginCredentials(admin_user, org,
 102                                                          admin_passwd))
 103             return client
 104
 105         except Exception as e:
 106             log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
 107
 108     def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
 109         vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
 110         return vdur['vim-id']
 111
 112     def get_vim_account(self, vim_account_id: str):
 113         """
 114            Method to get VIM account details by its ID
 115            arg - VIM ID
 116            return - dict with vim account details
 117         """
 118         vim_account = {}
 119         vim_account_info = self.auth_manager.get_credentials(vim_account_id)
 120
 121         vim_account['name'] = vim_account_info.name
 122         vim_account['vim_tenant_name'] = vim_account_info.tenant_name
 123         vim_account['vim_type'] = vim_account_info.type
 124         vim_account['vim_url'] = vim_account_info.url
 125         vim_account['org_user'] = vim_account_info.user
 126         vim_account['org_password'] = vim_account_info.password
 127         vim_account['vim_uuid'] = vim_account_info.uuid
 128
 129         vim_config = json.loads(vim_account_info.config)
 130         vim_account['admin_username'] = vim_config['admin_username']
 131         vim_account['admin_password'] = vim_config['admin_password']
 132         vim_account['vrops_site'] = vim_config['vrops_site']
 133         vim_account['vrops_user'] = vim_config['vrops_user']
 134         vim_account['vrops_password'] = vim_config['vrops_password']
 135         vim_account['vcenter_ip'] = vim_config['vcenter_ip']
 136         vim_account['vcenter_port'] = vim_config['vcenter_port']
 137         vim_account['vcenter_user'] = vim_config['vcenter_user']
 138         vim_account['vcenter_password'] = vim_config['vcenter_password']
 139
 140         if vim_config['nsx_manager'] is not None:
 141             vim_account['nsx_manager'] = vim_config['nsx_manager']
 142
 143         if vim_config['nsx_user'] is not None:
 144             vim_account['nsx_user'] = vim_config['nsx_user']
 145
 146         if vim_config['nsx_password'] is not None:
 147             vim_account['nsx_password'] = vim_config['nsx_password']
 148
 149         if vim_config['orgname'] is not None:
 150             vim_account['orgname'] = vim_config['orgname']
 151
 152         return vim_account
 153
 154     def get_vm_moref_id(self, vapp_uuid):
 155         """
 156            Method to get the moref_id of given VM
 157            arg - vapp_uuid
 158            return - VM mored_id
 159         """
 160         vm_moref_id = None
 161         try:
 162             if vapp_uuid:
 163                 vm_details = self.get_vapp_details_rest(vapp_uuid)
 164
 165                 if vm_details and "vm_vcenter_info" in vm_details:
 166                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
 167
 168             log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
 169             return vm_moref_id
 170
 171         except Exception as exp:
 172             log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc()))
 173
 174     def get_vapp_details_rest(self, vapp_uuid=None):
 175         """
 176         Method retrieve vapp detail from vCloud director
 177         vapp_uuid - is vapp identifier.
 178         Returns - VM MOref ID or return None
 179         """
 180         parsed_respond = {}
 181         vca = None
 182
 183         if vapp_uuid is None:
 184             return parsed_respond
 185
 186         vca = self.connect_as_admin()
 187
 188         if not vca:
 189             log.error("Failed to connect to vCD")
 190             return parsed_respond
 191
 192         url_list = [self.vcloud_site, '/api/vApp/vapp-', vapp_uuid]
 193         get_vapp_restcall = ''.join(url_list)
 194
 195         if vca._session:
 196             headers = {'Accept': 'application/*+xml;version=' + API_VERSION,
 197                        'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
 198             response = requests.get(get_vapp_restcall,
 199                                     headers=headers,
 200                                     verify=False)
 201
 202             if response.status_code != 200:
 203                 log.error("REST API call {} failed. Return status code {}".format(get_vapp_restcall,
 204                                                                                   response.content))
 205                 return parsed_respond
 206
 207             try:
 208                 xmlroot_respond = XmlElementTree.fromstring(response.content)
 209
 210                 namespaces = {'vm': 'http://www.vmware.com/vcloud/v1.5',
 211                               "vmext": "http://www.vmware.com/vcloud/extension/v1.5",
 212                               "xmlns": "http://www.vmware.com/vcloud/v1.5"}
 213
 214                 # parse children section for other attrib
 215                 children_section = xmlroot_respond.find('vm:Children/', namespaces)
 216                 if children_section is not None:
 217                     vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
 218                     if vCloud_extension_section is not None:
 219                         vm_vcenter_info = {}
 220                         vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
 221                         vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
 222                         if vmext is not None:
 223                             vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
 224                         parsed_respond["vm_vcenter_info"] = vm_vcenter_info
 225
 226             except Exception as exp:
 227                 log.info("Error occurred for getting vApp details: {}\n{}".format(exp,
 228                                                                                   traceback.format_exc())
 229                          )
 230
 231         return parsed_respond
 232
 233     def get_vm_resource_id(self, vm_moref_id):
 234         """ Find resource ID in vROPs using vm_moref_id
 235         """
 236         if vm_moref_id is None:
 237             return None
 238
 239         api_url = '/suite-api/api/resources?resourceKind=VirtualMachine'
 240         headers = {'Accept': 'application/json'}
 241
 242         resp = requests.get(self.vrops_site + api_url,
 243                             auth=(self.vrops_user, self.vrops_password),
 244                             verify=False, headers=headers)
 245
 246         if resp.status_code != 200:
 247             log.error("Failed to get resource details for{} {} {}".format(vm_moref_id,
 248                                                                           resp.status_code,
 249                                                                           resp.content))
 250             return None
 251
 252         vm_resource_id = None
 253         try:
 254             resp_data = json.loads(resp.content.decode('utf-8'))
 255             if resp_data.get('resourceList') is not None:
 256                 resource_list = resp_data.get('resourceList')
 257                 for resource in resource_list:
 258                     if resource.get('resourceKey') is not None:
 259                         resource_details = resource['resourceKey']
 260                         if resource_details.get('resourceIdentifiers') is not None:
 261                             resource_identifiers = resource_details['resourceIdentifiers']
 262                             for resource_identifier in resource_identifiers:
 263                                 if resource_identifier['identifierType']['name'] == 'VMEntityObjectID':
 264                                     if resource_identifier.get('value') is not None and \
 265                                             resource_identifier['value'] == vm_moref_id:
 266                                         vm_resource_id = resource['identifier']
 267                                         log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id,
 268                                                                                                        vm_moref_id))
 269
 270         except Exception as exp:
 271             log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc()))
 272
 273         return vm_resource_id
 274
 275     def collect(self, vnfr: dict):
 276         nsr_id = vnfr['nsr-id-ref']
 277         vnf_member_index = vnfr['member-vnf-index-ref']
 278         vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
 279         metrics = []
 280         for vdur in vnfr['vdur']:
 281             # This avoids errors when vdur records have not been completely filled
 282             if 'name' not in vdur:
 283                 continue
 284             vdu = next(
 285                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
 286
 287             )
 288             if 'monitoring-param' in vdu:
 289                 for param in vdu['monitoring-param']:
 290                     metric_name = param['nfvi-metric']
 291                     vrops_metric_name = METRIC_MAPPINGS[metric_name]
 292                     resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
 293
 294                     # Find vm_moref_id from vApp uuid in vCD
 295                     vm_moref_id = self.get_vm_moref_id(resource_uuid)
 296                     if vm_moref_id is None:
 297                         log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid))
 298                         return
 299
 300                     # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
 301                     resource_id = self.get_vm_resource_id(vm_moref_id)
 302                     if resource_id is None:
 303                         log.debug("Failed to find resource in vROPs: {}".format(resource_uuid))
 304                         return
 305                     try:
 306                         end_time = int(round(time.time() * 1000))
 307                         time_diff = PERIOD_MSEC['YEAR']
 308                         begin_time = end_time - time_diff
 309
 310                         api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format(
 311                             resource_id, vrops_metric_name, str(begin_time), str(end_time))
 312
 313                         headers = {'Accept': 'application/json'}
 314
 315                         resp = requests.get(self.vrops_site + api_url,
 316                                             auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
 317                                             )
 318
 319                         if resp.status_code != 200:
 320                             log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name,
 321                                                                                                  resp.status_code,
 322                                                                                                  resp.content))
 323                             return
 324
 325                         metrics_data = {}
 326                         m_data = json.loads(resp.content.decode('utf-8'))
 327
 328                         for resp_key, resp_val in six.iteritems(m_data):
 329                             if resp_key == 'values':
 330                                 data = m_data['values'][0]
 331                                 for data_k, data_v in six.iteritems(data):
 332                                     if data_k == 'stat-list':
 333                                         stat_list = data_v
 334                                         for stat_list_k, stat_list_v in six.iteritems(stat_list):
 335                                             for stat_keys, stat_vals in six.iteritems(stat_list_v[0]):
 336                                                 if stat_keys == 'timestamps':
 337                                                     metrics_data['time_series'] = stat_list_v[0]['timestamps']
 338                                                 if stat_keys == 'data':
 339                                                     metrics_data['metrics_series'] = stat_list_v[0]['data']
 340
 341                         if metrics_data:
 342                             metric = VnfMetric(nsr_id,
 343                                                vnf_member_index,
 344                                                vdur['name'],
 345                                                metric_name,
 346                                                metrics_data['metrics_series'][-1])
 347
 348                             metrics.append(metric)
 349
 350                     except Exception as e:
 351                         log.debug("No metric found: %s", e)
 352                         pass
 353         return metrics