X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_mon%2Fcollector%2Fvnf_collectors%2Fvmware.py;h=1bc33eb24fe3e385e7f3b493930ff77691493d50;hb=09bcf03e83c5863c3c0bcf47a457e70162fca94e;hp=00ef9ef2c4673f6dd93bbc845b80d0245247df5a;hpb=730dfd82ef49e2dc1f6d54bb8a1d5e877d4d50b2;p=osm%2FMON.git diff --git a/osm_mon/collector/vnf_collectors/vmware.py b/osm_mon/collector/vnf_collectors/vmware.py index 00ef9ef..1bc33eb 100644 --- a/osm_mon/collector/vnf_collectors/vmware.py +++ b/osm_mon/collector/vnf_collectors/vmware.py @@ -21,9 +21,7 @@ # contact: osslegalrouting@vmware.com ## -import json import logging -import time import traceback from xml.etree import ElementTree as XmlElementTree @@ -31,9 +29,8 @@ import requests from pyvcloud.vcd.client import BasicLoginCredentials from pyvcloud.vcd.client import Client -from osm_mon.collector.utils import CollectorUtils from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector -from osm_mon.collector.vnf_metric import VnfMetric +from osm_mon.collector.vnf_collectors.vrops.vrops_helper import vROPS_Helper from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config @@ -41,98 +38,18 @@ log = logging.getLogger(__name__) API_VERSION = '27.0' -TEN_MINUTES = 600000 - -# Ref: https://docs.vmware.com/en/vRealize-Operations-Manager/7.0/vrealize-operations-manager-70-reference-guide.pdf -# Potential metrics of interest -# "cpu|capacity_contentionPct" -# "cpu|corecount_provisioned" -# "cpu|costopPct" -# "cpu|demandmhz" -# "cpu|demandPct" -# "cpu|effective_limit" -# "cpu|iowaitPct" -# "cpu|readyPct" -# "cpu|swapwaitPct" -# "cpu|usage_average" -# "cpu|usagemhz_average" -# "cpu|usagemhz_average_mtd" -# "cpu|vm_capacity_provisioned" -# "cpu|workload" -# "guestfilesystem|percentage_total" -# "guestfilesystem|usage_total" -# "mem|consumedPct" -# "mem|guest_usage" -# "mem|host_contentionPct" -# "mem|reservation_used" -# "mem|swapinRate_average" -# "mem|swapoutRate_average" -# "mem|swapped_average" -# "mem|usage_average" -# "net:Aggregate of all instances|droppedPct" -# "net|broadcastTx_summation" -# "net|droppedTx_summation" -# "net|multicastTx_summation" -# "net|pnicBytesRx_average" -# "net|pnicBytesTx_average" -# "net|received_average" -# "net|transmitted_average" -# "net|usage_average" -# "virtualDisk:Aggregate of all instances|commandsAveraged_average" -# "virtualDisk:Aggregate of all instances|numberReadAveraged_average" -# "virtualDisk:Aggregate of all instances|numberWriteAveraged_average" -# "virtualDisk:Aggregate of all instances|totalLatency" -# "virtualDisk:Aggregate of all instances|totalReadLatency_average" -# "virtualDisk:Aggregate of all instances|totalWriteLatency_average" -# "virtualDisk:Aggregate of all instances|usage" -# "virtualDisk:Aggregate of all instances|vDiskOIO" -# "virtualDisk|read_average" -# "virtualDisk|write_average" - -METRIC_MAPPINGS = { - # Percent guest operating system active memory - "average_memory_utilization": "mem|usage_average", - # Percentage of CPU that was used out of all the CPU that was allocated - "cpu_utilization": "cpu|usage_average", - # KB/s of data read in the performance interval - "disk_read_bytes": "virtualDisk|read_average", - # Average of read commands per second during the collection interval. - "disk_read_ops": "virtualDisk:aggregate of all instances|numberReadAveraged_average", - # KB/s of data written in the performance interval - "disk_write_bytes": "virtualDisk|write_average", - # Average of write commands per second during the collection interval. - "disk_write_ops": "virtualDisk:aggregate of all instances|numberWriteAveraged_average", - # "packets_in_dropped": "net|droppedRx_summation", # Not supported by vROPS - # Transmitted packets dropped in the collection interval - "packets_out_dropped": "net|droppedTx_summation", - # Bytes received in the performance interval - "packets_received": "net|received_average", - # Packets transmitted in the performance interval - "packets_sent": "net|transmitted_average", -} - -# If the unit from vROPS does not align with the expected value. multiply by the specified amount to ensure -# the correct unit is returned. -METRIC_MULTIPLIERS = { - "disk_read_bytes": 1024, - "disk_write_bytes": 1024, - "net_bytes_received": 1024, - "net_bytes_sent": 1024 -} - class VMwareCollector(BaseVimCollector): def __init__(self, config: Config, vim_account_id: str): super().__init__(config, vim_account_id) self.common_db = CommonDbClient(config) vim_account = self.get_vim_account(vim_account_id) - self.vrops_site = vim_account['vrops_site'] - self.vrops_user = vim_account['vrops_user'] - self.vrops_password = vim_account['vrops_password'] self.vcloud_site = vim_account['vim_url'] self.admin_username = vim_account['admin_username'] self.admin_password = vim_account['admin_password'] - self.vim_uuid = vim_account['vim_uuid'] + self.vrops = vROPS_Helper(vrops_site=vim_account['vrops_site'], + vrops_user=vim_account['vrops_user'], + vrops_password=vim_account['vrops_password']) def connect_as_admin(self): """ Method connect as pvdc admin user to vCloud director. @@ -143,7 +60,7 @@ class VMwareCollector(BaseVimCollector): The return client object that letter can be used to connect to vcloud direct as admin for provider vdc """ - log.info("Logging into vCD org as admin.") + log.debug("Logging into vCD org as admin.") admin_user = None try: @@ -158,11 +75,7 @@ class VMwareCollector(BaseVimCollector): return client except Exception as e: - log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e)) - - def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str: - vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name) - return vdur['vim-id'] + log.error("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e)) def get_vim_account(self, vim_account_id: str): """ @@ -171,38 +84,16 @@ class VMwareCollector(BaseVimCollector): return - dict with vim account details """ vim_account = {} - vim_account_info = CollectorUtils.get_credentials(vim_account_id) + vim_account_info = self.common_db.get_vim_account(vim_account_id) - vim_account['name'] = vim_account_info.name - vim_account['vim_tenant_name'] = vim_account_info.tenant_name - vim_account['vim_type'] = vim_account_info.type - vim_account['vim_url'] = vim_account_info.url - vim_account['org_user'] = vim_account_info.user - vim_account['org_password'] = vim_account_info.password - vim_account['vim_uuid'] = vim_account_info.uuid + vim_account['vim_url'] = vim_account_info['vim_url'] - vim_config = json.loads(vim_account_info.config) + vim_config = vim_account_info['config'] vim_account['admin_username'] = vim_config['admin_username'] vim_account['admin_password'] = vim_config['admin_password'] vim_account['vrops_site'] = vim_config['vrops_site'] vim_account['vrops_user'] = vim_config['vrops_user'] vim_account['vrops_password'] = vim_config['vrops_password'] - vim_account['vcenter_ip'] = vim_config['vcenter_ip'] - vim_account['vcenter_port'] = vim_config['vcenter_port'] - vim_account['vcenter_user'] = vim_config['vcenter_user'] - vim_account['vcenter_password'] = vim_config['vcenter_password'] - - if vim_config['nsx_manager'] is not None: - vim_account['nsx_manager'] = vim_config['nsx_manager'] - - if vim_config['nsx_user'] is not None: - vim_account['nsx_user'] = vim_config['nsx_user'] - - if vim_config['nsx_password'] is not None: - vim_account['nsx_password'] = vim_config['nsx_password'] - - if vim_config['orgname'] is not None: - vim_account['orgname'] = vim_config['orgname'] return vim_account @@ -219,12 +110,14 @@ class VMwareCollector(BaseVimCollector): if vm_details and "vm_vcenter_info" in vm_details: vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None) - - log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid)) - return vm_moref_id + log.debug("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid)) + else: + log.error("Failed to find vm_moref_id from vApp UUID: {}".format(vapp_uuid)) except Exception as exp: - log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc())) + log.warning("Error occurred while getting VM moref ID for VM: {}\n{}".format(exp, traceback.format_exc())) + + return vm_moref_id def get_vapp_details_rest(self, vapp_uuid=None): """ @@ -278,137 +171,60 @@ class VMwareCollector(BaseVimCollector): parsed_respond["vm_vcenter_info"] = vm_vcenter_info except Exception as exp: - log.info("Error occurred for getting vApp details: {}\n{}".format(exp, - traceback.format_exc()) - ) + log.warning("Error occurred for getting vApp details: {}\n{}".format(exp, + traceback.format_exc())) return parsed_respond - def get_vm_resource_id(self, vm_moref_id): - """ Find resource ID in vROPs using vm_moref_id - """ - if vm_moref_id is None: - return None - - api_url = '/suite-api/api/resources?resourceKind=VirtualMachine' - headers = {'Accept': 'application/json'} - - resp = requests.get(self.vrops_site + api_url, - auth=(self.vrops_user, self.vrops_password), - verify=False, headers=headers) - - if resp.status_code != 200: - log.error("Failed to get resource details for{} {} {}".format(vm_moref_id, - resp.status_code, - resp.content)) - return None - - vm_resource_id = None - try: - resp_data = json.loads(resp.content.decode('utf-8')) - if resp_data.get('resourceList') is not None: - resource_list = resp_data.get('resourceList') - for resource in resource_list: - if resource.get('resourceKey') is not None: - resource_details = resource['resourceKey'] - if resource_details.get('resourceIdentifiers') is not None: - resource_identifiers = resource_details['resourceIdentifiers'] - for resource_identifier in resource_identifiers: - if resource_identifier['identifierType']['name'] == 'VMEntityObjectID': - if resource_identifier.get('value') is not None and \ - resource_identifier['value'] == vm_moref_id: - vm_resource_id = resource['identifier'] - log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id, - vm_moref_id)) + def collect(self, vnfr: dict): + vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) + vdu_mappings = {} - except Exception as exp: - log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc())) + # Populate extra tags for metrics + nsr_id = vnfr['nsr-id-ref'] + tags = {} + tags['ns_name'] = self.common_db.get_nsr(nsr_id)['name'] + if vnfr['_admin']['projects_read']: + tags['project_id'] = vnfr['_admin']['projects_read'][0] + else: + tags['project_id'] = '' - return vm_resource_id + # Fetch the list of all known resources from vROPS. + resource_list = self.vrops.get_vm_resource_list_from_vrops() - def collect(self, vnfr: dict): - nsr_id = vnfr['nsr-id-ref'] - vnf_member_index = vnfr['member-vnf-index-ref'] - vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) - metrics = [] for vdur in vnfr['vdur']: # This avoids errors when vdur records have not been completely filled if 'name' not in vdur: continue vdu = next( filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu']) - ) - if 'monitoring-param' not in vdu: + if 'monitoring-parameter' not in vdu: continue - resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name']) - + resource_uuid = vdur['vim-id'] # Find vm_moref_id from vApp uuid in vCD - vm_moref_id = self.get_vm_moref_id(resource_uuid) - if vm_moref_id is None: - log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid)) - continue - - # Based on vm_moref_id, find VM's corresponding resource_id in vROPs - resource_id = self.get_vm_resource_id(vm_moref_id) - if resource_id is None: - log.debug("Failed to find resource in vROPs: {}".format(resource_uuid)) + vim_id = self.get_vm_moref_id(resource_uuid) + if vim_id is None: + log.debug("Failed to find vROPS ID for vApp in vCD: {}".format(resource_uuid)) continue - stat_key = "" - monitoring_params = [] - for metric_entry in vdu['monitoring-param']: - metric_name = metric_entry['nfvi-metric'] - if metric_name not in METRIC_MAPPINGS: - log.debug("Metric {} not supported, ignoring".format(metric_name)) - continue - monitoring_params.append(metric_name) - vrops_metric_name = METRIC_MAPPINGS[metric_name] - stat_key = "{}&statKey={}".format(stat_key, vrops_metric_name) - - try: - end_time = int(round(time.time() * 1000)) - begin_time = end_time - TEN_MINUTES - - api_url = "/suite-api/api/resources/stats?resourceId={}&begin={}&end={}{}".format( - resource_id, str(begin_time), str(end_time), stat_key) - headers = {'Accept': 'application/json'} - - resp = requests.get(self.vrops_site + api_url, - auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers - ) - - if resp.status_code != 200: - log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vdur.name, - resp.status_code, - resp.content)) - continue - - m_data = json.loads(resp.content.decode('utf-8')) - - stat_list = m_data['values'][0]['stat-list']['stat'] - for item in stat_list: - reported_metric = item['statKey']['key'] - if reported_metric not in METRIC_MAPPINGS.values(): - continue - - metric_name = list(METRIC_MAPPINGS.keys())[list(METRIC_MAPPINGS.values()). - index(reported_metric)] - if metric_name in monitoring_params: - metric_value = item['data'][-1] - if metric_name in METRIC_MULTIPLIERS: - metric_value *= METRIC_MULTIPLIERS[metric_name] - metric = VnfMetric(nsr_id, - vnf_member_index, - vdur['name'], - metric_name, - metric_value) - - metrics.append(metric) - - except Exception as e: - log.debug("No metric found for {}: %s".format(vdur['name']), e) - pass - return metrics + vdu_mappings[vim_id] = {'name': vdur['name']} + + # Map the vROPS instance id to the vim-id so we can look it up. + for resource in resource_list: + for resourceIdentifier in resource['resourceKey']['resourceIdentifiers']: + if resourceIdentifier['identifierType']['name'] == 'VMEntityObjectID': + if resourceIdentifier['value'] != vim_id: + continue + vdu_mappings[vim_id]['vrops_id'] = resource['identifier'] + + if len(vdu_mappings) != 0: + return self.vrops.get_metrics(vdu_mappings=vdu_mappings, + monitoring_params=vdu['monitoring-parameter'], + vnfr=vnfr, + tags=tags + ) + else: + return []