Collect null project_ids as empty strings
[osm/MON.git] / osm_mon / collector / vnf_collectors / vmware.py
index 438c40c..b974d76 100644 (file)
 # contact:  osslegalrouting@vmware.com
 ##
 
 # contact:  osslegalrouting@vmware.com
 ##
 
-import json
 import logging
 import logging
-import time
 import traceback
 from xml.etree import ElementTree as XmlElementTree
 
 import requests
 import traceback
 from xml.etree import ElementTree as XmlElementTree
 
 import requests
-import six
 from pyvcloud.vcd.client import BasicLoginCredentials
 from pyvcloud.vcd.client import Client
 
 from pyvcloud.vcd.client import BasicLoginCredentials
 from pyvcloud.vcd.client import Client
 
-from osm_mon.collector.utils import CollectorUtils
 from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
 from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
-from osm_mon.collector.vnf_metric import VnfMetric
+from osm_mon.collector.vnf_collectors.vrops.vrops_helper import vROPS_Helper
 from osm_mon.core.common_db import CommonDbClient
 from osm_mon.core.config import Config
 
 from osm_mon.core.common_db import CommonDbClient
 from osm_mon.core.config import Config
 
@@ -42,39 +38,18 @@ log = logging.getLogger(__name__)
 
 API_VERSION = '27.0'
 
 
 API_VERSION = '27.0'
 
-PERIOD_MSEC = {'HR': 3600000,
-               'DAY': 86400000,
-               'WEEK': 604800000,
-               'MONTH': 2678400000,
-               'YEAR': 31536000000}
-
-METRIC_MAPPINGS = {
-    "average_memory_utilization": "mem|usage_average",
-    "cpu_utilization": "cpu|usage_average",
-    "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average",
-    "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average",
-    "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average",
-    "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average",
-    "packets_dropped_0": "net:4000|dropped",
-    "packets_dropped_1": "net:4001|dropped",
-    "packets_dropped_2": "net:4002|dropped",
-    "packets_received": "net:Aggregate of all instances|packetsRxPerSec",
-    "packets_sent": "net:Aggregate of all instances|packetsTxPerSec",
-}
-
 
 class VMwareCollector(BaseVimCollector):
     def __init__(self, config: Config, vim_account_id: str):
         super().__init__(config, vim_account_id)
         self.common_db = CommonDbClient(config)
         vim_account = self.get_vim_account(vim_account_id)
 
 class VMwareCollector(BaseVimCollector):
     def __init__(self, config: Config, vim_account_id: str):
         super().__init__(config, vim_account_id)
         self.common_db = CommonDbClient(config)
         vim_account = self.get_vim_account(vim_account_id)
-        self.vrops_site = vim_account['vrops_site']
-        self.vrops_user = vim_account['vrops_user']
-        self.vrops_password = vim_account['vrops_password']
         self.vcloud_site = vim_account['vim_url']
         self.admin_username = vim_account['admin_username']
         self.admin_password = vim_account['admin_password']
         self.vcloud_site = vim_account['vim_url']
         self.admin_username = vim_account['admin_username']
         self.admin_password = vim_account['admin_password']
-        self.vim_uuid = vim_account['vim_uuid']
+        self.vrops = vROPS_Helper(vrops_site=vim_account['vrops_site'],
+                                  vrops_user=vim_account['vrops_user'],
+                                  vrops_password=vim_account['vrops_password'])
 
     def connect_as_admin(self):
         """ Method connect as pvdc admin user to vCloud director.
 
     def connect_as_admin(self):
         """ Method connect as pvdc admin user to vCloud director.
@@ -85,7 +60,7 @@ class VMwareCollector(BaseVimCollector):
                 The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
         """
 
                 The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
         """
 
-        log.info("Logging into vCD org as admin.")
+        log.debug("Logging into vCD org as admin.")
 
         admin_user = None
         try:
 
         admin_user = None
         try:
@@ -100,11 +75,7 @@ class VMwareCollector(BaseVimCollector):
             return client
 
         except Exception as e:
             return client
 
         except Exception as e:
-            log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
-
-    def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
-        vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
-        return vdur['vim-id']
+            log.error("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
 
     def get_vim_account(self, vim_account_id: str):
         """
 
     def get_vim_account(self, vim_account_id: str):
         """
@@ -113,38 +84,16 @@ class VMwareCollector(BaseVimCollector):
            return - dict with vim account details
         """
         vim_account = {}
            return - dict with vim account details
         """
         vim_account = {}
-        vim_account_info = CollectorUtils.get_credentials(vim_account_id)
+        vim_account_info = self.common_db.get_vim_account(vim_account_id)
 
 
-        vim_account['name'] = vim_account_info.name
-        vim_account['vim_tenant_name'] = vim_account_info.tenant_name
-        vim_account['vim_type'] = vim_account_info.type
-        vim_account['vim_url'] = vim_account_info.url
-        vim_account['org_user'] = vim_account_info.user
-        vim_account['org_password'] = vim_account_info.password
-        vim_account['vim_uuid'] = vim_account_info.uuid
+        vim_account['vim_url'] = vim_account_info['vim_url']
 
 
-        vim_config = json.loads(vim_account_info.config)
+        vim_config = vim_account_info['config']
         vim_account['admin_username'] = vim_config['admin_username']
         vim_account['admin_password'] = vim_config['admin_password']
         vim_account['vrops_site'] = vim_config['vrops_site']
         vim_account['vrops_user'] = vim_config['vrops_user']
         vim_account['vrops_password'] = vim_config['vrops_password']
         vim_account['admin_username'] = vim_config['admin_username']
         vim_account['admin_password'] = vim_config['admin_password']
         vim_account['vrops_site'] = vim_config['vrops_site']
         vim_account['vrops_user'] = vim_config['vrops_user']
         vim_account['vrops_password'] = vim_config['vrops_password']
-        vim_account['vcenter_ip'] = vim_config['vcenter_ip']
-        vim_account['vcenter_port'] = vim_config['vcenter_port']
-        vim_account['vcenter_user'] = vim_config['vcenter_user']
-        vim_account['vcenter_password'] = vim_config['vcenter_password']
-
-        if vim_config['nsx_manager'] is not None:
-            vim_account['nsx_manager'] = vim_config['nsx_manager']
-
-        if vim_config['nsx_user'] is not None:
-            vim_account['nsx_user'] = vim_config['nsx_user']
-
-        if vim_config['nsx_password'] is not None:
-            vim_account['nsx_password'] = vim_config['nsx_password']
-
-        if vim_config['orgname'] is not None:
-            vim_account['orgname'] = vim_config['orgname']
 
         return vim_account
 
 
         return vim_account
 
@@ -161,12 +110,14 @@ class VMwareCollector(BaseVimCollector):
 
                 if vm_details and "vm_vcenter_info" in vm_details:
                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
 
                 if vm_details and "vm_vcenter_info" in vm_details:
                     vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
-
-            log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
-            return vm_moref_id
+                    log.debug("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
+                else:
+                    log.error("Failed to find vm_moref_id from vApp UUID: {}".format(vapp_uuid))
 
         except Exception as exp:
 
         except Exception as exp:
-            log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc()))
+            log.warning("Error occurred while getting VM moref ID for VM: {}\n{}".format(exp, traceback.format_exc()))
+
+        return vm_moref_id
 
     def get_vapp_details_rest(self, vapp_uuid=None):
         """
 
     def get_vapp_details_rest(self, vapp_uuid=None):
         """
@@ -175,7 +126,6 @@ class VMwareCollector(BaseVimCollector):
         Returns - VM MOref ID or return None
         """
         parsed_respond = {}
         Returns - VM MOref ID or return None
         """
         parsed_respond = {}
-        vca = None
 
         if vapp_uuid is None:
             return parsed_respond
 
         if vapp_uuid is None:
             return parsed_respond
@@ -221,130 +171,60 @@ class VMwareCollector(BaseVimCollector):
                         parsed_respond["vm_vcenter_info"] = vm_vcenter_info
 
             except Exception as exp:
                         parsed_respond["vm_vcenter_info"] = vm_vcenter_info
 
             except Exception as exp:
-                log.info("Error occurred for getting vApp details: {}\n{}".format(exp,
-                                                                                  traceback.format_exc())
-                         )
+                log.warning("Error occurred for getting vApp details: {}\n{}".format(exp,
+                                                                                     traceback.format_exc()))
 
         return parsed_respond
 
 
         return parsed_respond
 
-    def get_vm_resource_id(self, vm_moref_id):
-        """ Find resource ID in vROPs using vm_moref_id
-        """
-        if vm_moref_id is None:
-            return None
-
-        api_url = '/suite-api/api/resources?resourceKind=VirtualMachine'
-        headers = {'Accept': 'application/json'}
-
-        resp = requests.get(self.vrops_site + api_url,
-                            auth=(self.vrops_user, self.vrops_password),
-                            verify=False, headers=headers)
-
-        if resp.status_code != 200:
-            log.error("Failed to get resource details for{} {} {}".format(vm_moref_id,
-                                                                          resp.status_code,
-                                                                          resp.content))
-            return None
-
-        vm_resource_id = None
-        try:
-            resp_data = json.loads(resp.content.decode('utf-8'))
-            if resp_data.get('resourceList') is not None:
-                resource_list = resp_data.get('resourceList')
-                for resource in resource_list:
-                    if resource.get('resourceKey') is not None:
-                        resource_details = resource['resourceKey']
-                        if resource_details.get('resourceIdentifiers') is not None:
-                            resource_identifiers = resource_details['resourceIdentifiers']
-                            for resource_identifier in resource_identifiers:
-                                if resource_identifier['identifierType']['name'] == 'VMEntityObjectID':
-                                    if resource_identifier.get('value') is not None and \
-                                            resource_identifier['value'] == vm_moref_id:
-                                        vm_resource_id = resource['identifier']
-                                        log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id,
-                                                                                                       vm_moref_id))
+    def collect(self, vnfr: dict):
+        vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
+        vdu_mappings = {}
 
 
-        except Exception as exp:
-            log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc()))
+        # Populate extra tags for metrics
+        nsr_id = vnfr['nsr-id-ref']
+        tags = {}
+        tags['ns_name'] = self.common_db.get_nsr(nsr_id)['name']
+        if vnfr['_admin']['projects_read']:
+            tags['project_id'] = vnfr['_admin']['projects_read'][0]
+        else:
+            tags['project_id'] = ''
 
 
-        return vm_resource_id
+        # Fetch the list of all known resources from vROPS.
+        resource_list = self.vrops.get_vm_resource_list_from_vrops()
 
 
-    def collect(self, vnfr: dict):
-        nsr_id = vnfr['nsr-id-ref']
-        vnf_member_index = vnfr['member-vnf-index-ref']
-        vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
-        metrics = []
         for vdur in vnfr['vdur']:
             # This avoids errors when vdur records have not been completely filled
             if 'name' not in vdur:
                 continue
             vdu = next(
                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
         for vdur in vnfr['vdur']:
             # This avoids errors when vdur records have not been completely filled
             if 'name' not in vdur:
                 continue
             vdu = next(
                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
-
             )
             )
-            if 'monitoring-param' in vdu:
-                for param in vdu['monitoring-param']:
-                    metric_name = param['nfvi-metric']
-                    vrops_metric_name = METRIC_MAPPINGS[metric_name]
-                    resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
-
-                    # Find vm_moref_id from vApp uuid in vCD
-                    vm_moref_id = self.get_vm_moref_id(resource_uuid)
-                    if vm_moref_id is None:
-                        log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid))
-                        return
-
-                    # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
-                    resource_id = self.get_vm_resource_id(vm_moref_id)
-                    if resource_id is None:
-                        log.debug("Failed to find resource in vROPs: {}".format(resource_uuid))
-                        return
-                    try:
-                        end_time = int(round(time.time() * 1000))
-                        time_diff = PERIOD_MSEC['YEAR']
-                        begin_time = end_time - time_diff
-
-                        api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format(
-                            resource_id, vrops_metric_name, str(begin_time), str(end_time))
-
-                        headers = {'Accept': 'application/json'}
-
-                        resp = requests.get(self.vrops_site + api_url,
-                                            auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
-                                            )
-
-                        if resp.status_code != 200:
-                            log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name,
-                                                                                                 resp.status_code,
-                                                                                                 resp.content))
-                            return
-
-                        metrics_data = {}
-                        m_data = json.loads(resp.content.decode('utf-8'))
-
-                        for resp_key, resp_val in six.iteritems(m_data):
-                            if resp_key == 'values':
-                                data = m_data['values'][0]
-                                for data_k, data_v in six.iteritems(data):
-                                    if data_k == 'stat-list':
-                                        stat_list = data_v
-                                        for stat_list_k, stat_list_v in six.iteritems(stat_list):
-                                            for stat_keys, stat_vals in six.iteritems(stat_list_v[0]):
-                                                if stat_keys == 'timestamps':
-                                                    metrics_data['time_series'] = stat_list_v[0]['timestamps']
-                                                if stat_keys == 'data':
-                                                    metrics_data['metrics_series'] = stat_list_v[0]['data']
-
-                        if metrics_data:
-                            metric = VnfMetric(nsr_id,
-                                               vnf_member_index,
-                                               vdur['name'],
-                                               metric_name,
-                                               metrics_data['metrics_series'][-1])
-
-                            metrics.append(metric)
-
-                    except Exception as e:
-                        log.debug("No metric found: %s", e)
-                        pass
-        return metrics
+
+            if 'monitoring-param' not in vdu:
+                continue
+
+            resource_uuid = vdur['vim-id']
+            # Find vm_moref_id from vApp uuid in vCD
+            vim_id = self.get_vm_moref_id(resource_uuid)
+            if vim_id is None:
+                log.debug("Failed to find vROPS ID for vApp in vCD: {}".format(resource_uuid))
+                continue
+
+            vdu_mappings[vim_id] = {'name': vdur['name']}
+
+            # Map the vROPS instance id to the vim-id so we can look it up.
+            for resource in resource_list:
+                for resourceIdentifier in resource['resourceKey']['resourceIdentifiers']:
+                    if resourceIdentifier['identifierType']['name'] == 'VMEntityObjectID':
+                        if resourceIdentifier['value'] != vim_id:
+                            continue
+                        vdu_mappings[vim_id]['vrops_id'] = resource['identifier']
+
+        if len(vdu_mappings) != 0:
+            return self.vrops.get_metrics(vdu_mappings=vdu_mappings,
+                                          monitoring_params=vdu['monitoring-param'],
+                                          vnfr=vnfr,
+                                          tags=tags
+                                          )
+        else:
+            return []