Fixes VIO collection

[osm/MON.git] / osm_mon / collector / vnf_collectors / vmware.py

diff --git a/osm_mon/collector/vnf_collectors/vmware.py b/osm_mon/collector/vnf_collectors/vmware.py
index e79ce06..329d61d 100644 (file)
--- a/osm_mon/collector/vnf_collectors/vmware.py
+++ b/osm_mon/collector/vnf_collectors/vmware.py
@@ -23,7 +23,6 @@
 
 import json
 import logging
 
 import json
 import logging

-import time

 import traceback
 from xml.etree import ElementTree as XmlElementTree
 
 import traceback
 from xml.etree import ElementTree as XmlElementTree
 


@@ -33,106 +32,26 @@ from pyvcloud.vcd.client import Client
 
 from osm_mon.collector.utils.collector import CollectorUtils
 from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
 
 from osm_mon.collector.utils.collector import CollectorUtils
 from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector

-from osm_mon.collector.vnf_metric import VnfMetric

 from osm_mon.core.common_db import CommonDbClient
 from osm_mon.core.config import Config
 from osm_mon.core.common_db import CommonDbClient
 from osm_mon.core.config import Config

+from osm_mon.collector.vnf_collectors.vrops.vrops_helper import vROPS_Helper

 
 log = logging.getLogger(__name__)
 
 API_VERSION = '27.0'
 
 
 log = logging.getLogger(__name__)
 
 API_VERSION = '27.0'
 

-TEN_MINUTES = 600000
-
-# Ref: https://docs.vmware.com/en/vRealize-Operations-Manager/7.0/vrealize-operations-manager-70-reference-guide.pdf
-# Potential metrics of interest
-# "cpu|capacity_contentionPct"
-# "cpu|corecount_provisioned"
-# "cpu|costopPct"
-# "cpu|demandmhz"
-# "cpu|demandPct"
-# "cpu|effective_limit"
-# "cpu|iowaitPct"
-# "cpu|readyPct"
-# "cpu|swapwaitPct"
-# "cpu|usage_average"
-# "cpu|usagemhz_average"
-# "cpu|usagemhz_average_mtd"
-# "cpu|vm_capacity_provisioned"
-# "cpu|workload"
-# "guestfilesystem|percentage_total"
-# "guestfilesystem|usage_total"
-# "mem|consumedPct"
-# "mem|guest_usage"
-# "mem|host_contentionPct"
-# "mem|reservation_used"
-# "mem|swapinRate_average"
-# "mem|swapoutRate_average"
-# "mem|swapped_average"
-# "mem|usage_average"
-# "net:Aggregate of all instances|droppedPct"
-# "net|broadcastTx_summation"
-# "net|droppedTx_summation"
-# "net|multicastTx_summation"
-# "net|pnicBytesRx_average"
-# "net|pnicBytesTx_average"
-# "net|received_average"
-# "net|transmitted_average"
-# "net|usage_average"
-# "virtualDisk:Aggregate of all instances|commandsAveraged_average"
-# "virtualDisk:Aggregate of all instances|numberReadAveraged_average"
-# "virtualDisk:Aggregate of all instances|numberWriteAveraged_average"
-# "virtualDisk:Aggregate of all instances|totalLatency"
-# "virtualDisk:Aggregate of all instances|totalReadLatency_average"
-# "virtualDisk:Aggregate of all instances|totalWriteLatency_average"
-# "virtualDisk:Aggregate of all instances|usage"
-# "virtualDisk:Aggregate of all instances|vDiskOIO"
-# "virtualDisk|read_average"
-# "virtualDisk|write_average"
-
-METRIC_MAPPINGS = {
-    # Percent guest operating system active memory
-    "average_memory_utilization": "mem|usage_average",
-    # Percentage of CPU that was used out of all the CPU that was allocated
-    "cpu_utilization": "cpu|usage_average",
-    # KB/s of data read in the performance interval
-    "disk_read_bytes": "virtualDisk|read_average",
-    # Average of read commands per second during the collection interval.
-    "disk_read_ops": "virtualDisk:aggregate of all instances|numberReadAveraged_average",
-    # KB/s  of data written in the performance interval
-    "disk_write_bytes": "virtualDisk|write_average",
-    # Average of write commands per second during the collection interval.
-    "disk_write_ops": "virtualDisk:aggregate of all instances|numberWriteAveraged_average",
-    # "packets_in_dropped": "net|droppedRx_summation",  # Not supported by vROPS
-    # Transmitted packets dropped in the collection interval
-    "packets_out_dropped": "net|droppedTx_summation",
-    # Bytes received in the performance interval
-    "packets_received": "net|received_average",
-    # Packets transmitted in the performance interval
-    "packets_sent": "net|transmitted_average",
-}
-
-# If the unit from vROPS does not align with the expected value. multiply by the specified amount to ensure
-# the correct unit is returned.
-METRIC_MULTIPLIERS = {
-    "disk_read_bytes": 1024,
-    "disk_write_bytes": 1024,
-    "packets_received": 1024,
-    "packets_sent": 1024
-}
-

 
 class VMwareCollector(BaseVimCollector):
     def __init__(self, config: Config, vim_account_id: str):
         super().__init__(config, vim_account_id)
         self.common_db = CommonDbClient(config)
         vim_account = self.get_vim_account(vim_account_id)
 
 class VMwareCollector(BaseVimCollector):
     def __init__(self, config: Config, vim_account_id: str):
         super().__init__(config, vim_account_id)
         self.common_db = CommonDbClient(config)
         vim_account = self.get_vim_account(vim_account_id)

-        self.vrops_site = vim_account['vrops_site']
-        self.vrops_user = vim_account['vrops_user']
-        self.vrops_password = vim_account['vrops_password']

         self.vcloud_site = vim_account['vim_url']
         self.admin_username = vim_account['admin_username']
         self.admin_password = vim_account['admin_password']
         self.vcloud_site = vim_account['vim_url']
         self.admin_username = vim_account['admin_username']
         self.admin_password = vim_account['admin_password']

-        self.vim_uuid = vim_account['vim_uuid']
+        self.vrops = vROPS_Helper(vrops_site=vim_account['vrops_site'],
+                                  vrops_user=vim_account['vrops_user'],
+                                  vrops_password=vim_account['vrops_password'])

 
     def connect_as_admin(self):
         """ Method connect as pvdc admin user to vCloud director.
 
     def connect_as_admin(self):
         """ Method connect as pvdc admin user to vCloud director.


@@ -143,7 +62,7 @@ class VMwareCollector(BaseVimCollector):
                 The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
         """
 
                 The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
         """
 

-        log.info("Logging into vCD org as admin.")
+        log.debug("Logging into vCD org as admin.")

 
         admin_user = None
         try:
 
         admin_user = None
         try:


@@ -158,11 +77,7 @@ class VMwareCollector(BaseVimCollector):
             return client
 
         except Exception as e:
             return client
 
         except Exception as e:

-            log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
-
-    def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
-        vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
-        return vdur['vim-id']
+            log.error("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))

 
     def get_vim_account(self, vim_account_id: str):
         """
 
     def get_vim_account(self, vim_account_id: str):
         """


@@ -173,13 +88,7 @@ class VMwareCollector(BaseVimCollector):
         vim_account = {}
         vim_account_info = CollectorUtils.get_credentials(vim_account_id)
 
         vim_account = {}
         vim_account_info = CollectorUtils.get_credentials(vim_account_id)
 

-        vim_account['name'] = vim_account_info.name
-        vim_account['vim_tenant_name'] = vim_account_info.tenant_name
-        vim_account['vim_type'] = vim_account_info.type

         vim_account['vim_url'] = vim_account_info.url
         vim_account['vim_url'] = vim_account_info.url

-        vim_account['org_user'] = vim_account_info.user
-        vim_account['org_password'] = vim_account_info.password
-        vim_account['vim_uuid'] = vim_account_info.uuid

 
         vim_config = json.loads(vim_account_info.config)
         vim_account['admin_username'] = vim_config['admin_username']
 
         vim_config = json.loads(vim_account_info.config)
         vim_account['admin_username'] = vim_config['admin_username']


@@ -187,22 +96,6 @@ class VMwareCollector(BaseVimCollector):
         vim_account['vrops_site'] = vim_config['vrops_site']
         vim_account['vrops_user'] = vim_config['vrops_user']
         vim_account['vrops_password'] = vim_config['vrops_password']
         vim_account['vrops_site'] = vim_config['vrops_site']
         vim_account['vrops_user'] = vim_config['vrops_user']
         vim_account['vrops_password'] = vim_config['vrops_password']

-        vim_account['vcenter_ip'] = vim_config['vcenter_ip']
-        vim_account['vcenter_port'] = vim_config['vcenter_port']
-        vim_account['vcenter_user'] = vim_config['vcenter_user']
-        vim_account['vcenter_password'] = vim_config['vcenter_password']
-
-        if vim_config['nsx_manager'] is not None:
-            vim_account['nsx_manager'] = vim_config['nsx_manager']
-
-        if vim_config['nsx_user'] is not None:
-            vim_account['nsx_user'] = vim_config['nsx_user']
-
-        if vim_config['nsx_password'] is not None:
-            vim_account['nsx_password'] = vim_config['nsx_password']
-
-        if vim_config['orgname'] is not None:
-            vim_account['orgname'] = vim_config['orgname']

 
         return vim_account
 
 
         return vim_account
 


@@ -219,12 +112,14 @@ class VMwareCollector(BaseVimCollector):
 
                 if vm_details and "vm_vcenter_info" in vm_details:
                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
 
                 if vm_details and "vm_vcenter_info" in vm_details:
                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)

-
-            log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
-            return vm_moref_id
+                    log.debug("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
+                else:
+                    log.error("Failed to find vm_moref_id from vApp UUID: {}".format(vapp_uuid))

 
         except Exception as exp:
 
         except Exception as exp:

-            log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc()))
+            log.warning("Error occurred while getting VM moref ID for VM: {}\n{}".format(exp, traceback.format_exc()))
+
+        return vm_moref_id

 
     def get_vapp_details_rest(self, vapp_uuid=None):
         """
 
     def get_vapp_details_rest(self, vapp_uuid=None):
         """


@@ -278,137 +173,49 @@ class VMwareCollector(BaseVimCollector):
                         parsed_respond["vm_vcenter_info"] = vm_vcenter_info
 
             except Exception as exp:
                         parsed_respond["vm_vcenter_info"] = vm_vcenter_info
 
             except Exception as exp:

-                log.info("Error occurred for getting vApp details: {}\n{}".format(exp,
-                                                                                  traceback.format_exc())
-                         )
+                log.warning("Error occurred for getting vApp details: {}\n{}".format(exp,
+                                                                                     traceback.format_exc()))

 
         return parsed_respond
 
 
         return parsed_respond
 

-    def get_vm_resource_id(self, vm_moref_id):
-        """ Find resource ID in vROPs using vm_moref_id
-        """
-        if vm_moref_id is None:
-            return None
-
-        api_url = '/suite-api/api/resources?resourceKind=VirtualMachine'
-        headers = {'Accept': 'application/json'}
-
-        resp = requests.get(self.vrops_site + api_url,
-                            auth=(self.vrops_user, self.vrops_password),
-                            verify=False, headers=headers)
-
-        if resp.status_code != 200:
-            log.error("Failed to get resource details for{} {} {}".format(vm_moref_id,
-                                                                          resp.status_code,
-                                                                          resp.content))
-            return None
-
-        vm_resource_id = None
-        try:
-            resp_data = json.loads(resp.content.decode('utf-8'))
-            if resp_data.get('resourceList') is not None:
-                resource_list = resp_data.get('resourceList')
-                for resource in resource_list:
-                    if resource.get('resourceKey') is not None:
-                        resource_details = resource['resourceKey']
-                        if resource_details.get('resourceIdentifiers') is not None:
-                            resource_identifiers = resource_details['resourceIdentifiers']
-                            for resource_identifier in resource_identifiers:
-                                if resource_identifier['identifierType']['name'] == 'VMEntityObjectID':
-                                    if resource_identifier.get('value') is not None and \
-                                            resource_identifier['value'] == vm_moref_id:
-                                        vm_resource_id = resource['identifier']
-                                        log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id,
-                                                                                                       vm_moref_id))
-
-        except Exception as exp:
-            log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc()))
-
-        return vm_resource_id
-

     def collect(self, vnfr: dict):
     def collect(self, vnfr: dict):

-        nsr_id = vnfr['nsr-id-ref']
-        vnf_member_index = vnfr['member-vnf-index-ref']

         vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
         vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])

-        metrics = []
+        vdu_mappings = {}
+
+        # Fetch the list of all known resources from vROPS.
+        resource_list = self.vrops.get_vm_resource_list_from_vrops()
+

         for vdur in vnfr['vdur']:
             # This avoids errors when vdur records have not been completely filled
             if 'name' not in vdur:
                 continue
             vdu = next(
                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
         for vdur in vnfr['vdur']:
             # This avoids errors when vdur records have not been completely filled
             if 'name' not in vdur:
                 continue
             vdu = next(
                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])

-

             )
 
             if 'monitoring-param' not in vdu:
                 continue
 
             )
 
             if 'monitoring-param' not in vdu:
                 continue
 

-            resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
-
+            resource_uuid = vdur['vim-id']

             # Find vm_moref_id from vApp uuid in vCD
             # Find vm_moref_id from vApp uuid in vCD

-            vm_moref_id = self.get_vm_moref_id(resource_uuid)
-            if vm_moref_id is None:
-                log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid))
-                continue
-
-            # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
-            resource_id = self.get_vm_resource_id(vm_moref_id)
-            if resource_id is None:
-                log.debug("Failed to find resource in vROPs: {}".format(resource_uuid))
+            vim_id = self.get_vm_moref_id(resource_uuid)
+            if vim_id is None:
+                log.debug("Failed to find vROPS ID for vApp in vCD: {}".format(resource_uuid))

                 continue
 
                 continue
 

-            stat_key = ""
-            monitoring_params = []
-            for metric_entry in vdu['monitoring-param']:
-                metric_name = metric_entry['nfvi-metric']
-                if metric_name not in METRIC_MAPPINGS:
-                    log.debug("Metric {} not supported, ignoring".format(metric_name))
-                    continue
-                monitoring_params.append(metric_name)
-                vrops_metric_name = METRIC_MAPPINGS[metric_name]
-                stat_key = "{}&statKey={}".format(stat_key, vrops_metric_name)
-
-            try:
-                end_time = int(round(time.time() * 1000))
-                begin_time = end_time - TEN_MINUTES
-
-                api_url = "/suite-api/api/resources/stats?resourceId={}&begin={}&end={}{}".format(
-                    resource_id, str(begin_time), str(end_time), stat_key)
-                headers = {'Accept': 'application/json'}
-
-                resp = requests.get(self.vrops_site + api_url,
-                                    auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
-                                    )
-
-                if resp.status_code != 200:
-                    log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vdur.name,
-                                                                                         resp.status_code,
-                                                                                         resp.content))
-                    continue
-
-                m_data = json.loads(resp.content.decode('utf-8'))
-
-                stat_list = m_data['values'][0]['stat-list']['stat']
-                for item in stat_list:
-                    reported_metric = item['statKey']['key']
-                    if reported_metric not in METRIC_MAPPINGS.values():
-                        continue
-
-                    metric_name = list(METRIC_MAPPINGS.keys())[list(METRIC_MAPPINGS.values()).
-                                                               index(reported_metric)]
-                    if metric_name in monitoring_params:
-                        metric_value = item['data'][-1]
-                        if metric_name in METRIC_MULTIPLIERS:
-                            metric_value *= METRIC_MULTIPLIERS[metric_name]
-                        metric = VnfMetric(nsr_id,
-                                           vnf_member_index,
-                                           vdur['name'],
-                                           metric_name,
-                                           metric_value)
-
-                        metrics.append(metric)
-
-            except Exception as e:
-                log.debug("No metric found for {}: %s".format(vdur['name']), e)
-                pass
-        return metrics
+            vdu_mappings[vim_id] = {'name': vdur['name']}
+
+            # Map the vROPS instance id to the vim-id so we can look it up.
+            for resource in resource_list:
+                for resourceIdentifier in resource['resourceKey']['resourceIdentifiers']:
+                    if resourceIdentifier['identifierType']['name'] == 'VMEntityObjectID':
+                        if resourceIdentifier['value'] != vim_id:
+                            continue
+                        vdu_mappings[vim_id]['vrops_id'] = resource['identifier']
+
+        if len(vdu_mappings) != 0:
+            return self.vrops.get_metrics(vdu_mappings=vdu_mappings,
+                                          monitoring_params=vdu['monitoring-param'],
+                                          vnfr=vnfr)
+        else:
+            return []