osm_mon/collector/vnf_collectors/vmware.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2016-2017 VMware Inc.
   5 # This file is part of ETSI OSM
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact:  osslegalrouting@vmware.com
  22 ##
  23
  24 import json
  25 import logging
  26 import time
  27 import traceback
  28 from xml.etree import ElementTree as XmlElementTree
  29
  30 import requests
  31 import six
  32 from pyvcloud.vcd.client import BasicLoginCredentials
  33 from pyvcloud.vcd.client import Client
  34
  35 from osm_mon.collector.utils import CollectorUtils
  36 from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
  37 from osm_mon.collector.vnf_metric import VnfMetric
  38 from osm_mon.core.common_db import CommonDbClient
  39 from osm_mon.core.config import Config
  40
  41 log = logging.getLogger(__name__)
  42
  43 API_VERSION = '27.0'
  44
  45 PERIOD_MSEC = {'HR': 3600000,
  46                'DAY': 86400000,
  47                'WEEK': 604800000,
  48                'MONTH': 2678400000,
  49                'YEAR': 31536000000}
  50
  51 METRIC_MAPPINGS = {
  52     "average_memory_utilization": "mem|usage_average",
  53     "cpu_utilization": "cpu|usage_average",
  54     "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average",
  55     "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average",
  56     "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average",
  57     "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average",
  58     "packets_dropped_0": "net:4000|dropped",
  59     "packets_dropped_1": "net:4001|dropped",
  60     "packets_dropped_2": "net:4002|dropped",
  61     "packets_received": "net:Aggregate of all instances|packetsRxPerSec",
  62     "packets_sent": "net:Aggregate of all instances|packetsTxPerSec",
  63 }
  64
  65
  66 class VMwareCollector(BaseVimCollector):
  67     def __init__(self, config: Config, vim_account_id: str):
  68         super().__init__(config, vim_account_id)
  69         self.common_db = CommonDbClient(config)
  70         vim_account = self.get_vim_account(vim_account_id)
  71         self.vrops_site = vim_account['vrops_site']
  72         self.vrops_user = vim_account['vrops_user']
  73         self.vrops_password = vim_account['vrops_password']
  74         self.vcloud_site = vim_account['vim_url']
  75         self.admin_username = vim_account['admin_username']
  76         self.admin_password = vim_account['admin_password']
  77         self.vim_uuid = vim_account['vim_uuid']
  78
  79     def connect_as_admin(self):
  80         """ Method connect as pvdc admin user to vCloud director.
  81             There are certain action that can be done only by provider vdc admin user.
  82             Organization creation / provider network creation etc.
  83
  84             Returns:
  85                 The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
  86         """
  87
  88         log.info("Logging into vCD org as admin.")
  89
  90         admin_user = None
  91         try:
  92             host = self.vcloud_site
  93             admin_user = self.admin_username
  94             admin_passwd = self.admin_password
  95             org = 'System'
  96             client = Client(host, verify_ssl_certs=False)
  97             client.set_credentials(BasicLoginCredentials(admin_user, org,
  98                                                          admin_passwd))
  99             return client
 100
 101         except Exception as e:
 102             log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
 103
 104     def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
 105         vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
 106         return vdur['vim-id']
 107
 108     def get_vim_account(self, vim_account_id: str):
 109         """
 110            Method to get VIM account details by its ID
 111            arg - VIM ID
 112            return - dict with vim account details
 113         """
 114         vim_account = {}
 115         vim_account_info = CollectorUtils.get_credentials(vim_account_id)
 116
 117         vim_account['name'] = vim_account_info.name
 118         vim_account['vim_tenant_name'] = vim_account_info.tenant_name
 119         vim_account['vim_type'] = vim_account_info.type
 120         vim_account['vim_url'] = vim_account_info.url
 121         vim_account['org_user'] = vim_account_info.user
 122         vim_account['org_password'] = vim_account_info.password
 123         vim_account['vim_uuid'] = vim_account_info.uuid
 124
 125         vim_config = json.loads(vim_account_info.config)
 126         vim_account['admin_username'] = vim_config['admin_username']
 127         vim_account['admin_password'] = vim_config['admin_password']
 128         vim_account['vrops_site'] = vim_config['vrops_site']
 129         vim_account['vrops_user'] = vim_config['vrops_user']
 130         vim_account['vrops_password'] = vim_config['vrops_password']
 131         vim_account['vcenter_ip'] = vim_config['vcenter_ip']
 132         vim_account['vcenter_port'] = vim_config['vcenter_port']
 133         vim_account['vcenter_user'] = vim_config['vcenter_user']
 134         vim_account['vcenter_password'] = vim_config['vcenter_password']
 135
 136         if vim_config['nsx_manager'] is not None:
 137             vim_account['nsx_manager'] = vim_config['nsx_manager']
 138
 139         if vim_config['nsx_user'] is not None:
 140             vim_account['nsx_user'] = vim_config['nsx_user']
 141
 142         if vim_config['nsx_password'] is not None:
 143             vim_account['nsx_password'] = vim_config['nsx_password']
 144
 145         if vim_config['orgname'] is not None:
 146             vim_account['orgname'] = vim_config['orgname']
 147
 148         return vim_account
 149
 150     def get_vm_moref_id(self, vapp_uuid):
 151         """
 152            Method to get the moref_id of given VM
 153            arg - vapp_uuid
 154            return - VM mored_id
 155         """
 156         vm_moref_id = None
 157         try:
 158             if vapp_uuid:
 159                 vm_details = self.get_vapp_details_rest(vapp_uuid)
 160
 161                 if vm_details and "vm_vcenter_info" in vm_details:
 162                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
 163
 164             log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
 165             return vm_moref_id
 166
 167         except Exception as exp:
 168             log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc()))
 169
 170     def get_vapp_details_rest(self, vapp_uuid=None):
 171         """
 172         Method retrieve vapp detail from vCloud director
 173         vapp_uuid - is vapp identifier.
 174         Returns - VM MOref ID or return None
 175         """
 176         parsed_respond = {}
 177         vca = None
 178
 179         if vapp_uuid is None:
 180             return parsed_respond
 181
 182         vca = self.connect_as_admin()
 183
 184         if not vca:
 185             log.error("Failed to connect to vCD")
 186             return parsed_respond
 187
 188         url_list = [self.vcloud_site, '/api/vApp/vapp-', vapp_uuid]
 189         get_vapp_restcall = ''.join(url_list)
 190
 191         if vca._session:
 192             headers = {'Accept': 'application/*+xml;version=' + API_VERSION,
 193                        'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
 194             response = requests.get(get_vapp_restcall,
 195                                     headers=headers,
 196                                     verify=False)
 197
 198             if response.status_code != 200:
 199                 log.error("REST API call {} failed. Return status code {}".format(get_vapp_restcall,
 200                                                                                   response.content))
 201                 return parsed_respond
 202
 203             try:
 204                 xmlroot_respond = XmlElementTree.fromstring(response.content)
 205
 206                 namespaces = {'vm': 'http://www.vmware.com/vcloud/v1.5',
 207                               "vmext": "http://www.vmware.com/vcloud/extension/v1.5",
 208                               "xmlns": "http://www.vmware.com/vcloud/v1.5"}
 209
 210                 # parse children section for other attrib
 211                 children_section = xmlroot_respond.find('vm:Children/', namespaces)
 212                 if children_section is not None:
 213                     vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
 214                     if vCloud_extension_section is not None:
 215                         vm_vcenter_info = {}
 216                         vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
 217                         vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
 218                         if vmext is not None:
 219                             vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
 220                         parsed_respond["vm_vcenter_info"] = vm_vcenter_info
 221
 222             except Exception as exp:
 223                 log.info("Error occurred for getting vApp details: {}\n{}".format(exp,
 224                                                                                   traceback.format_exc())
 225                          )
 226
 227         return parsed_respond
 228
 229     def get_vm_resource_id(self, vm_moref_id):
 230         """ Find resource ID in vROPs using vm_moref_id
 231         """
 232         if vm_moref_id is None:
 233             return None
 234
 235         api_url = '/suite-api/api/resources?resourceKind=VirtualMachine'
 236         headers = {'Accept': 'application/json'}
 237
 238         resp = requests.get(self.vrops_site + api_url,
 239                             auth=(self.vrops_user, self.vrops_password),
 240                             verify=False, headers=headers)
 241
 242         if resp.status_code != 200:
 243             log.error("Failed to get resource details for{} {} {}".format(vm_moref_id,
 244                                                                           resp.status_code,
 245                                                                           resp.content))
 246             return None
 247
 248         vm_resource_id = None
 249         try:
 250             resp_data = json.loads(resp.content.decode('utf-8'))
 251             if resp_data.get('resourceList') is not None:
 252                 resource_list = resp_data.get('resourceList')
 253                 for resource in resource_list:
 254                     if resource.get('resourceKey') is not None:
 255                         resource_details = resource['resourceKey']
 256                         if resource_details.get('resourceIdentifiers') is not None:
 257                             resource_identifiers = resource_details['resourceIdentifiers']
 258                             for resource_identifier in resource_identifiers:
 259                                 if resource_identifier['identifierType']['name'] == 'VMEntityObjectID':
 260                                     if resource_identifier.get('value') is not None and \
 261                                             resource_identifier['value'] == vm_moref_id:
 262                                         vm_resource_id = resource['identifier']
 263                                         log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id,
 264                                                                                                        vm_moref_id))
 265
 266         except Exception as exp:
 267             log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc()))
 268
 269         return vm_resource_id
 270
 271     def collect(self, vnfr: dict):
 272         nsr_id = vnfr['nsr-id-ref']
 273         vnf_member_index = vnfr['member-vnf-index-ref']
 274         vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
 275         metrics = []
 276         for vdur in vnfr['vdur']:
 277             # This avoids errors when vdur records have not been completely filled
 278             if 'name' not in vdur:
 279                 continue
 280             vdu = next(
 281                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
 282
 283             )
 284             if 'monitoring-param' in vdu:
 285                 for param in vdu['monitoring-param']:
 286                     metric_name = param['nfvi-metric']
 287                     vrops_metric_name = METRIC_MAPPINGS[metric_name]
 288                     resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
 289
 290                     # Find vm_moref_id from vApp uuid in vCD
 291                     vm_moref_id = self.get_vm_moref_id(resource_uuid)
 292                     if vm_moref_id is None:
 293                         log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid))
 294                         return
 295
 296                     # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
 297                     resource_id = self.get_vm_resource_id(vm_moref_id)
 298                     if resource_id is None:
 299                         log.debug("Failed to find resource in vROPs: {}".format(resource_uuid))
 300                         return
 301                     try:
 302                         end_time = int(round(time.time() * 1000))
 303                         time_diff = PERIOD_MSEC['YEAR']
 304                         begin_time = end_time - time_diff
 305
 306                         api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format(
 307                             resource_id, vrops_metric_name, str(begin_time), str(end_time))
 308
 309                         headers = {'Accept': 'application/json'}
 310
 311                         resp = requests.get(self.vrops_site + api_url,
 312                                             auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
 313                                             )
 314
 315                         if resp.status_code != 200:
 316                             log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name,
 317                                                                                                  resp.status_code,
 318                                                                                                  resp.content))
 319                             return
 320
 321                         metrics_data = {}
 322                         m_data = json.loads(resp.content.decode('utf-8'))
 323
 324                         for resp_key, resp_val in six.iteritems(m_data):
 325                             if resp_key == 'values':
 326                                 data = m_data['values'][0]
 327                                 for data_k, data_v in six.iteritems(data):
 328                                     if data_k == 'stat-list':
 329                                         stat_list = data_v
 330                                         for stat_list_k, stat_list_v in six.iteritems(stat_list):
 331                                             for stat_keys, stat_vals in six.iteritems(stat_list_v[0]):
 332                                                 if stat_keys == 'timestamps':
 333                                                     metrics_data['time_series'] = stat_list_v[0]['timestamps']
 334                                                 if stat_keys == 'data':
 335                                                     metrics_data['metrics_series'] = stat_list_v[0]['data']
 336
 337                         if metrics_data:
 338                             metric = VnfMetric(nsr_id,
 339                                                vnf_member_index,
 340                                                vdur['name'],
 341                                                metric_name,
 342                                                metrics_data['metrics_series'][-1])
 343
 344                             metrics.append(metric)
 345
 346                     except Exception as e:
 347                         log.debug("No metric found: %s", e)
 348                         pass
 349         return metrics