Reduce Resource Consumption 98/7698/6
authorbeierl <mbeierl@vmware.com>
Mon, 24 Jun 2019 19:51:33 +0000 (15:51 -0400)
committerbeierl <mbeierl@vmware.com>
Fri, 28 Jun 2019 19:39:20 +0000 (15:39 -0400)
The vmware vnf collector requests a time series of data from
vROPS and then parses the whole block in order to retrieve the
most recent data element.  The time frame for the request
should be limited to a sane value, such as 1 hour.

The look ups for items that do not change should be outside
of the loop for looking up the metrics.

Add more metrics to list of supported metrics.

Improve error handling.

BUG 784

Change-Id: Ie3cef457eaf657c18ba682e3c5d3abf40a759e69
Signed-off-by: beierl <mbeierl@vmware.com>
osm_mon/collector/vnf_collectors/vmware.py
osm_mon/tests/unit/collector/vnf_collectors/vmware/__init__.py [new file with mode: 0644]
osm_mon/tests/unit/collector/vnf_collectors/vmware/mock_vcd.py [new file with mode: 0644]
osm_mon/tests/unit/collector/vnf_collectors/vmware/osm_mocks/VNFD.json [new file with mode: 0644]
osm_mon/tests/unit/collector/vnf_collectors/vmware/osm_mocks/VNFR.json [new file with mode: 0644]
osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py [new file with mode: 0644]
osm_mon/tests/unit/collector/vnf_collectors/vmware/vcd_mocks/OK.json [new file with mode: 0644]
osm_mon/tests/unit/collector/vnf_collectors/vmware/vcd_mocks/vrops_multi.json [new file with mode: 0644]
test-requirements.txt
tox.ini

index 438c40c..00ef9ef 100644 (file)
@@ -28,7 +28,6 @@ import traceback
 from xml.etree import ElementTree as XmlElementTree
 
 import requests
-import six
 from pyvcloud.vcd.client import BasicLoginCredentials
 from pyvcloud.vcd.client import Client
 
@@ -42,24 +41,83 @@ log = logging.getLogger(__name__)
 
 API_VERSION = '27.0'
 
-PERIOD_MSEC = {'HR': 3600000,
-               'DAY': 86400000,
-               'WEEK': 604800000,
-               'MONTH': 2678400000,
-               'YEAR': 31536000000}
+TEN_MINUTES = 600000
+
+# Ref: https://docs.vmware.com/en/vRealize-Operations-Manager/7.0/vrealize-operations-manager-70-reference-guide.pdf
+# Potential metrics of interest
+# "cpu|capacity_contentionPct"
+# "cpu|corecount_provisioned"
+# "cpu|costopPct"
+# "cpu|demandmhz"
+# "cpu|demandPct"
+# "cpu|effective_limit"
+# "cpu|iowaitPct"
+# "cpu|readyPct"
+# "cpu|swapwaitPct"
+# "cpu|usage_average"
+# "cpu|usagemhz_average"
+# "cpu|usagemhz_average_mtd"
+# "cpu|vm_capacity_provisioned"
+# "cpu|workload"
+# "guestfilesystem|percentage_total"
+# "guestfilesystem|usage_total"
+# "mem|consumedPct"
+# "mem|guest_usage"
+# "mem|host_contentionPct"
+# "mem|reservation_used"
+# "mem|swapinRate_average"
+# "mem|swapoutRate_average"
+# "mem|swapped_average"
+# "mem|usage_average"
+# "net:Aggregate of all instances|droppedPct"
+# "net|broadcastTx_summation"
+# "net|droppedTx_summation"
+# "net|multicastTx_summation"
+# "net|pnicBytesRx_average"
+# "net|pnicBytesTx_average"
+# "net|received_average"
+# "net|transmitted_average"
+# "net|usage_average"
+# "virtualDisk:Aggregate of all instances|commandsAveraged_average"
+# "virtualDisk:Aggregate of all instances|numberReadAveraged_average"
+# "virtualDisk:Aggregate of all instances|numberWriteAveraged_average"
+# "virtualDisk:Aggregate of all instances|totalLatency"
+# "virtualDisk:Aggregate of all instances|totalReadLatency_average"
+# "virtualDisk:Aggregate of all instances|totalWriteLatency_average"
+# "virtualDisk:Aggregate of all instances|usage"
+# "virtualDisk:Aggregate of all instances|vDiskOIO"
+# "virtualDisk|read_average"
+# "virtualDisk|write_average"
 
 METRIC_MAPPINGS = {
+    # Percent guest operating system active memory
     "average_memory_utilization": "mem|usage_average",
+    # Percentage of CPU that was used out of all the CPU that was allocated
     "cpu_utilization": "cpu|usage_average",
-    "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average",
-    "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average",
-    "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average",
-    "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average",
-    "packets_dropped_0": "net:4000|dropped",
-    "packets_dropped_1": "net:4001|dropped",
-    "packets_dropped_2": "net:4002|dropped",
-    "packets_received": "net:Aggregate of all instances|packetsRxPerSec",
-    "packets_sent": "net:Aggregate of all instances|packetsTxPerSec",
+    # KB/s of data read in the performance interval
+    "disk_read_bytes": "virtualDisk|read_average",
+    # Average of read commands per second during the collection interval.
+    "disk_read_ops": "virtualDisk:aggregate of all instances|numberReadAveraged_average",
+    # KB/s  of data written in the performance interval
+    "disk_write_bytes": "virtualDisk|write_average",
+    # Average of write commands per second during the collection interval.
+    "disk_write_ops": "virtualDisk:aggregate of all instances|numberWriteAveraged_average",
+    # "packets_in_dropped": "net|droppedRx_summation",  # Not supported by vROPS
+    # Transmitted packets dropped in the collection interval
+    "packets_out_dropped": "net|droppedTx_summation",
+    # Bytes received in the performance interval
+    "packets_received": "net|received_average",
+    # Packets transmitted in the performance interval
+    "packets_sent": "net|transmitted_average",
+}
+
+# If the unit from vROPS does not align with the expected value. multiply by the specified amount to ensure
+# the correct unit is returned.
+METRIC_MULTIPLIERS = {
+    "disk_read_bytes": 1024,
+    "disk_write_bytes": 1024,
+    "net_bytes_received": 1024,
+    "net_bytes_sent": 1024
 }
 
 
@@ -175,7 +233,6 @@ class VMwareCollector(BaseVimCollector):
         Returns - VM MOref ID or return None
         """
         parsed_respond = {}
-        vca = None
 
         if vapp_uuid is None:
             return parsed_respond
@@ -282,69 +339,76 @@ class VMwareCollector(BaseVimCollector):
                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
 
             )
-            if 'monitoring-param' in vdu:
-                for param in vdu['monitoring-param']:
-                    metric_name = param['nfvi-metric']
-                    vrops_metric_name = METRIC_MAPPINGS[metric_name]
-                    resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
-
-                    # Find vm_moref_id from vApp uuid in vCD
-                    vm_moref_id = self.get_vm_moref_id(resource_uuid)
-                    if vm_moref_id is None:
-                        log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid))
-                        return
-
-                    # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
-                    resource_id = self.get_vm_resource_id(vm_moref_id)
-                    if resource_id is None:
-                        log.debug("Failed to find resource in vROPs: {}".format(resource_uuid))
-                        return
-                    try:
-                        end_time = int(round(time.time() * 1000))
-                        time_diff = PERIOD_MSEC['YEAR']
-                        begin_time = end_time - time_diff
-
-                        api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format(
-                            resource_id, vrops_metric_name, str(begin_time), str(end_time))
-
-                        headers = {'Accept': 'application/json'}
-
-                        resp = requests.get(self.vrops_site + api_url,
-                                            auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
-                                            )
-
-                        if resp.status_code != 200:
-                            log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name,
-                                                                                                 resp.status_code,
-                                                                                                 resp.content))
-                            return
-
-                        metrics_data = {}
-                        m_data = json.loads(resp.content.decode('utf-8'))
-
-                        for resp_key, resp_val in six.iteritems(m_data):
-                            if resp_key == 'values':
-                                data = m_data['values'][0]
-                                for data_k, data_v in six.iteritems(data):
-                                    if data_k == 'stat-list':
-                                        stat_list = data_v
-                                        for stat_list_k, stat_list_v in six.iteritems(stat_list):
-                                            for stat_keys, stat_vals in six.iteritems(stat_list_v[0]):
-                                                if stat_keys == 'timestamps':
-                                                    metrics_data['time_series'] = stat_list_v[0]['timestamps']
-                                                if stat_keys == 'data':
-                                                    metrics_data['metrics_series'] = stat_list_v[0]['data']
-
-                        if metrics_data:
-                            metric = VnfMetric(nsr_id,
-                                               vnf_member_index,
-                                               vdur['name'],
-                                               metric_name,
-                                               metrics_data['metrics_series'][-1])
-
-                            metrics.append(metric)
-
-                    except Exception as e:
-                        log.debug("No metric found: %s", e)
-                        pass
+
+            if 'monitoring-param' not in vdu:
+                continue
+
+            resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
+
+            # Find vm_moref_id from vApp uuid in vCD
+            vm_moref_id = self.get_vm_moref_id(resource_uuid)
+            if vm_moref_id is None:
+                log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid))
+                continue
+
+            # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
+            resource_id = self.get_vm_resource_id(vm_moref_id)
+            if resource_id is None:
+                log.debug("Failed to find resource in vROPs: {}".format(resource_uuid))
+                continue
+
+            stat_key = ""
+            monitoring_params = []
+            for metric_entry in vdu['monitoring-param']:
+                metric_name = metric_entry['nfvi-metric']
+                if metric_name not in METRIC_MAPPINGS:
+                    log.debug("Metric {} not supported, ignoring".format(metric_name))
+                    continue
+                monitoring_params.append(metric_name)
+                vrops_metric_name = METRIC_MAPPINGS[metric_name]
+                stat_key = "{}&statKey={}".format(stat_key, vrops_metric_name)
+
+            try:
+                end_time = int(round(time.time() * 1000))
+                begin_time = end_time - TEN_MINUTES
+
+                api_url = "/suite-api/api/resources/stats?resourceId={}&begin={}&end={}{}".format(
+                    resource_id, str(begin_time), str(end_time), stat_key)
+                headers = {'Accept': 'application/json'}
+
+                resp = requests.get(self.vrops_site + api_url,
+                                    auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
+                                    )
+
+                if resp.status_code != 200:
+                    log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vdur.name,
+                                                                                         resp.status_code,
+                                                                                         resp.content))
+                    continue
+
+                m_data = json.loads(resp.content.decode('utf-8'))
+
+                stat_list = m_data['values'][0]['stat-list']['stat']
+                for item in stat_list:
+                    reported_metric = item['statKey']['key']
+                    if reported_metric not in METRIC_MAPPINGS.values():
+                        continue
+
+                    metric_name = list(METRIC_MAPPINGS.keys())[list(METRIC_MAPPINGS.values()).
+                                                               index(reported_metric)]
+                    if metric_name in monitoring_params:
+                        metric_value = item['data'][-1]
+                        if metric_name in METRIC_MULTIPLIERS:
+                            metric_value *= METRIC_MULTIPLIERS[metric_name]
+                        metric = VnfMetric(nsr_id,
+                                           vnf_member_index,
+                                           vdur['name'],
+                                           metric_name,
+                                           metric_value)
+
+                        metrics.append(metric)
+
+            except Exception as e:
+                log.debug("No metric found for {}: %s".format(vdur['name']), e)
+                pass
         return metrics
diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/__init__.py b/osm_mon/tests/unit/collector/vnf_collectors/vmware/__init__.py
new file mode 100644 (file)
index 0000000..30d864e
--- /dev/null
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# #
+# Copyright 2016-2019 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact:  osslegalrouting@vmware.com
+# #
diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/mock_vcd.py b/osm_mon/tests/unit/collector/vnf_collectors/vmware/mock_vcd.py
new file mode 100644 (file)
index 0000000..6938873
--- /dev/null
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 VMware
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to VMware
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+#         http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: mbeierl@vmware.com
+# #
+
+import os
+import re
+
+
+def mock_vdc_response(mocker, method='GET', site='https://vrops', url_pattern='', response_file='OK'):
+    '''Helper function to load a canned response from a file.'''
+    with open(os.path.join(os.path.dirname(__file__), 'vcd_mocks',
+                           '%s' % response_file), 'r') as f:
+        response = f.read()
+
+    matcher = re.compile(site + url_pattern)
+    mocker.register_uri(method, matcher, text=response)
diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/osm_mocks/VNFD.json b/osm_mon/tests/unit/collector/vnf_collectors/vmware/osm_mocks/VNFD.json
new file mode 100644 (file)
index 0000000..ed34789
--- /dev/null
@@ -0,0 +1,123 @@
+{
+       "_admin": {
+               "created": 1561567611.6193357,
+               "modified": 1561567611.6193357,
+               "onboardingState": "ONBOARDED",
+               "operationalState": "ENABLED",
+               "projects_read": [
+                       "admin"
+               ],
+               "projects_write": [
+                       "admin"
+               ],
+               "storage": {
+                       "descriptor": "cirros_vnf/cirros_vdu_alarm_vnfd.yaml",
+                       "folder": "9d116df6-6fa7-4a5b-b284-a67f554c1261",
+                       "fs": "local",
+                       "path": "/app/storage/",
+                       "pkg-dir": "cirros_vnf",
+                       "zipfile": "cirros_vdu_alarm_vnfd.tar.gz"
+               },
+               "type": "vnfd",
+               "usageState": "NOT_IN_USE",
+               "userDefinedData": {
+               }
+       },
+       "_id": "9d116df6-6fa7-4a5b-b284-a67f554c1261",
+       "connection-point": [
+               {
+                       "name": "eth0",
+                       "type": "VPORT"
+               }
+       ],
+       "description": "Simple VNF example with a cirros and a VNF alarm",
+       "id": "cirros_vdu_alarm_vnf",
+       "logo": "cirros-64.png",
+       "mgmt-interface": {
+               "cp": "eth0"
+       },
+       "monitoring-param": [
+               {
+                       "aggregation-type": "AVERAGE",
+                       "id": "cirros_vnf_cpu_util",
+                       "name": "cirros_vnf_cpu_util",
+                       "vdu-monitoring-param": {
+                               "vdu-monitoring-param-ref": "cirros_vnfd-VM_cpu_util",
+                               "vdu-ref": "cirros_vnfd-VM"
+                       }
+               },
+               {
+                       "aggregation-type": "AVERAGE",
+                       "id": "cirros_vnf_average_memory_utilization",
+                       "name": "cirros_vnf_average_memory_utilization",
+                       "vdu-monitoring-param": {
+                               "vdu-monitoring-param-ref": "cirros_vnfd-VM_average_memory_utilization",
+                               "vdu-ref": "cirros_vnfd-VM"
+                       }
+               }
+       ],
+       "name": "cirros_vdu_alarm_vnf",
+       "short-name": "cirros_vdu_alarm_vnf",
+       "vdu": [
+               {
+                       "alarm": [
+                               {
+                                       "actions": {
+                                               "alarm": [
+                                                       {
+                                                               "url": "https://webhook.site/e65ad8a0-53b5-4d7f-bd50-c1a50b260871"
+                                                       }
+                                               ],
+                                               "insufficient-data": [
+                                                       {
+                                                               "url": "https://webhook.site/e65ad8a0-53b5-4d7f-bd50-c1a50b260871"
+                                                       }
+                                               ],
+                                               "ok": [
+                                                       {
+                                                               "url": "https://webhook.site/e65ad8a0-53b5-4d7f-bd50-c1a50b260871"
+                                                       }
+                                               ]
+                                       },
+                                       "alarm-id": "alarm-1",
+                                       "operation": "LT",
+                                       "value": "20.0000",
+                                       "vnf-monitoring-param-ref": "cirros_vnf_cpu_util"
+                               }
+                       ],
+                       "count": "1",
+                       "description": "cirros_vnfd-VM",
+                       "id": "cirros_vnfd-VM",
+                       "image": "ubuntu",
+                       "interface": [
+                               {
+                                       "external-connection-point-ref": "eth0",
+                                       "name": "eth0",
+                                       "virtual-interface": {
+                                               "bandwidth": "0",
+                                               "type": "VIRTIO",
+                                               "vpci": "0000:00:0a.0"
+                                       }
+                               }
+                       ],
+                       "monitoring-param": [
+                               {
+                                       "id": "cirros_vnfd-VM_cpu_util",
+                                       "nfvi-metric": "cpu_utilization"
+                               },
+                               {
+                                       "id": "cirros_vnfd-VM_average_memory_utilization",
+                                       "nfvi-metric": "average_memory_utilization"
+                               }
+                       ],
+                       "name": "cirros_vnfd-VM",
+                       "vm-flavor": {
+                               "memory-mb": "256",
+                               "storage-gb": "2",
+                               "vcpu-count": 1
+                       }
+               }
+       ],
+       "vendor": "OSM",
+       "version": "1.0"
+}
diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/osm_mocks/VNFR.json b/osm_mon/tests/unit/collector/vnf_collectors/vmware/osm_mocks/VNFR.json
new file mode 100644 (file)
index 0000000..332949b
--- /dev/null
@@ -0,0 +1,52 @@
+{
+       "_admin": {
+               "created": 1561569715.5365124,
+               "modified": 1561569715.5365124,
+               "projects_read": [
+                       "admin"
+               ],
+               "projects_write": [
+                       "admin"
+               ]
+       },
+       "_id": "c222366b-4e67-47e8-a3a5-d602ef9f953f",
+       "additionalParamsForVnf": null,
+       "connection-point": [
+               {
+                       "connection-point-id": null,
+                       "id": null,
+                       "name": "eth0"
+               }
+       ],
+       "created-time": 1561569715.5344007,
+       "id": "c222366b-4e67-47e8-a3a5-d602ef9f953f",
+       "ip-address": "172.21.6.212",
+       "member-vnf-index-ref": "1",
+       "nsr-id-ref": "66300531-3fb2-43e2-a988-51a1afd9a8f2",
+       "vdur": [
+               {
+                       "_id": "ddb46f3c-d586-4a8e-80c3-87a74d0a3b37",
+                       "count-index": 0,
+                       "interfaces": [
+                               {
+                                       "ip-address": "172.21.6.212",
+                                       "mac-address": "00:50:56:03:01:79",
+                                       "mgmt-vnf": true,
+                                       "name": "eth0",
+                                       "ns-vld-id": "cirros_nsd_vld1"
+                               }
+                       ],
+                       "internal-connection-point": [
+                       ],
+                       "ip-address": "172.21.6.212",
+                       "name": "mb-cirros-1-cirros_vnfd-VM-1",
+                       "status": "ACTIVE",
+                       "status-detailed": "ACTIVE",
+                       "vdu-id-ref": "cirros_vnfd-VM",
+                       "vim-id": "962e29e7-e0b0-4253-b3ef-88df7971f12e"
+               }
+       ],
+       "vim-account-id": "43d176fd-dc6d-4fb0-acf3-faf03b1d2589",
+       "vnfd-id": "9d116df6-6fa7-4a5b-b284-a67f554c1261",
+       "vnfd-ref": "cirros_vdu_alarm_vnf"
+}
diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py b/osm_mon/tests/unit/collector/vnf_collectors/vmware/test_vcd_collector.py
new file mode 100644 (file)
index 0000000..19d1496
--- /dev/null
@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+# #
+# Copyright 2016-2019 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact:  osslegalrouting@vmware.com
+# #
+
+from osm_mon.collector.vnf_collectors.vmware import VMwareCollector
+from osm_mon.core.config import Config
+from osm_mon.tests.unit.collector.vnf_collectors.vmware.mock_vcd import mock_vdc_response
+from unittest import TestCase, mock
+
+import json
+import os
+import requests_mock
+
+VIM_ACCOUNT = {"vrops_site": "https://vrops",
+               "vrops_user": "",
+               "vrops_password": "",
+               "vim_url": "",
+               "admin_username": "",
+               "admin_password": "",
+               "vim_uuid": ""}
+
+
+@mock.patch.object(VMwareCollector, 'get_vm_resource_id',
+                   spec_set=True, autospec=True)
+@mock.patch.object(VMwareCollector, 'get_vm_moref_id',
+                   spec_set=True, autospec=True)
+class CollectorTest(TestCase):
+
+    @mock.patch.object(VMwareCollector, 'get_vim_account',
+                       spec_set=True, autospec=True)
+    @mock.patch('osm_mon.collector.vnf_collectors.vmware.CommonDbClient')
+    def setUp(self, mock_db, mock_get_vim_account):
+        super().setUp()
+        mock_get_vim_account.return_value = VIM_ACCOUNT
+        self.collector = VMwareCollector(Config(), "9de6df67-b820-48c3-bcae-ee4838c5c5f4")
+        self.mock_db = mock_db
+        with open(os.path.join(os.path.dirname(__file__), 'osm_mocks', 'VNFR.json'), 'r') as f:
+            self.vnfr = json.load(f)
+        with open(os.path.join(os.path.dirname(__file__), 'osm_mocks', 'VNFD.json'), 'r') as f:
+            self.vnfd = json.load(f)
+
+    def tearDown(self):
+        super().tearDown()
+
+    def test_collect_cpu_and_memory(self, mock_vm_moref_id, mock_vm_resource_id):
+
+        mock_vm_moref_id.return_value = "moref"
+        mock_vm_resource_id.return_value = "resource"
+        self.mock_db.return_value.get_vnfd.return_value = self.vnfd
+
+        with requests_mock.Mocker() as mock_requests:
+            mock_vdc_response(mock_requests,
+                              url_pattern='/suite-api/api/resources/stats.*',
+                              response_file='vrops_multi.json')
+            metrics = self.collector.collect(self.vnfr)
+            self.assertEqual(len(metrics), 2, "Number of metrics returned")
+            self.assertEqual(metrics[0].name, "cpu_utilization", "First metric name")
+            self.assertEqual(metrics[1].name, "average_memory_utilization", "Second metric name")
+            self.assertEqual(metrics[0].value, 100.0, "CPU metric value")
+            self.assertEqual(metrics[1].value, 20.515941619873047, "Memory metric value")
+
+    def test_collect_one_metric_only(self, mock_vm_moref_id, mock_vm_resource_id):
+
+        mock_vm_moref_id.return_value = "moref"
+        mock_vm_resource_id.return_value = "resource"
+
+        self.vnfd['vdu'][0]['monitoring-param'] = [
+            {'id': 'cirros_vnfd-VM_cpu_util', 'nfvi-metric': 'cpu_utilization'},
+            ]
+        self.mock_db.return_value.get_vnfd.return_value = self.vnfd
+
+        with requests_mock.Mocker() as mock_requests:
+            mock_vdc_response(mock_requests,
+                              url_pattern='/suite-api/api/resources/stats.*',
+                              response_file='vrops_multi.json')
+            metrics = self.collector.collect(self.vnfr)
+            self.assertEqual(len(metrics), 1, "Number of metrics returned")
+            self.assertEqual(metrics[0].name, "cpu_utilization", "First metric name")
+            self.assertEqual(metrics[0].value, 100.0, "CPU metric value")
+
+    def test_collect_adjusted_metric(self, mock_vm_moref_id, mock_vm_resource_id):
+
+        mock_vm_moref_id.return_value = "moref"
+        mock_vm_resource_id.return_value = "resource"
+
+        self.vnfd['vdu'][0]['monitoring-param'] = [
+            {'id': 'cirros_vnfd-VM_cpu_util', 'nfvi-metric': 'disk_read_bytes'},
+            ]
+        self.mock_db.return_value.get_vnfd.return_value = self.vnfd
+
+        with requests_mock.Mocker() as mock_requests:
+            mock_vdc_response(mock_requests,
+                              url_pattern='/suite-api/api/resources/stats.*',
+                              response_file='vrops_multi.json')
+            metrics = self.collector.collect(self.vnfr)
+            self.assertEqual(len(metrics), 1, "Number of metrics returned")
+            self.assertEqual(metrics[0].name, "disk_read_bytes", "First metric name")
+            self.assertEqual(metrics[0].value, 10240.0, "Disk read bytes (not KB/s)")
+
+    def test_collect_not_provided_metric(self, mock_vm_moref_id, mock_vm_resource_id):
+
+        mock_vm_moref_id.return_value = "moref"
+        mock_vm_resource_id.return_value = "resource"
+
+        self.vnfd['vdu'][0]['monitoring-param'] = [
+            {'id': 'cirros_vnfd-VM_packets_sent', 'nfvi-metric': 'packets_sent'},
+            ]
+        self.mock_db.return_value.get_vnfd.return_value = self.vnfd
+
+        with requests_mock.Mocker() as mock_requests:
+            mock_vdc_response(mock_requests,
+                              url_pattern='/suite-api/api/resources/stats.*',
+                              response_file='OK.json')
+            metrics = self.collector.collect(self.vnfr)
+            self.assertEqual(len(metrics), 0, "Number of metrics returned")
+
+    def test_collect_unkown_metric(self, mock_vm_moref_id, mock_vm_resource_id):
+
+        mock_vm_moref_id.return_value = "moref"
+        mock_vm_resource_id.return_value = "resource"
+
+        self.vnfd['vdu'][0]['monitoring-param'] = [
+            {'id': 'cirros_vnfd-Unknown_Metric', 'nfvi-metric': 'unknown'},
+            ]
+        self.mock_db.return_value.get_vnfd.return_value = self.vnfd
+
+        with requests_mock.Mocker() as mock_requests:
+            mock_vdc_response(mock_requests,
+                              url_pattern='/suite-api/api/resources/stats.*',
+                              response_file='vrops_multi.json')
+            metrics = self.collector.collect(self.vnfr)
+            self.assertEqual(len(metrics), 0, "Number of metrics returned")
+
+    def test_collect_vrops_error(self, mock_vm_moref_id, mock_vm_resource_id):
+
+        mock_vm_moref_id.return_value = "moref"
+        mock_vm_resource_id.return_value = "resource"
+        self.mock_db.return_value.get_vnfd.return_value = self.vnfd
+
+        with requests_mock.Mocker():
+            metrics = self.collector.collect(self.vnfr)
+            self.assertEqual(len(metrics), 0, "Number of metrics returned")
diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/vcd_mocks/OK.json b/osm_mon/tests/unit/collector/vnf_collectors/vmware/vcd_mocks/OK.json
new file mode 100644 (file)
index 0000000..0967ef4
--- /dev/null
@@ -0,0 +1 @@
+{}
diff --git a/osm_mon/tests/unit/collector/vnf_collectors/vmware/vcd_mocks/vrops_multi.json b/osm_mon/tests/unit/collector/vnf_collectors/vmware/vcd_mocks/vrops_multi.json
new file mode 100644 (file)
index 0000000..95b0305
--- /dev/null
@@ -0,0 +1,251 @@
+{
+       "values": [
+               {
+                       "resourceId": "a761f4ae-40a3-4696-ae37-ee447ed421c3",
+                       "stat-list": {
+                               "stat": [
+                                       {
+                                               "data": [
+                                                       0.0,
+                                                       0.0,
+                                                       0.0,
+                                                       0.0
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "virtualDisk:aggregate of all instances|totalReadLatency_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       1.1373332738876343,
+                                                       1.1440000534057617,
+                                                       1.1239999532699585,
+                                                       100.0
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "cpu|usage_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       6.0,
+                                                       6.066666603088379,
+                                                       6.133333206176758,
+                                                       6.066666603088379
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "virtualDisk:aggregate of all instances|numberWriteAveraged_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       0.0,
+                                                       0.0,
+                                                       0.0,
+                                                       0.0
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "virtualDisk:aggregate of all instances|numberReadAveraged_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       0.0,
+                                                       0.0,
+                                                       0.0,
+                                                       0.0
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "net|droppedTx_summation"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       0.20000000298023224,
+                                                       0.20000000298023224,
+                                                       0.20000000298023224,
+                                                       10.0
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "virtualDisk|read_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       0.0,
+                                                       0.0,
+                                                       0.0,
+                                                       0.0
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "net|transmitted_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       20.491540908813477,
+                                                       20.505388259887695,
+                                                       20.517423629760742,
+                                                       20.515941619873047
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "mem|usage_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       0.0,
+                                                       0.0,
+                                                       0.0,
+                                                       0.0
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "virtualDisk:aggregate of all instances|totalWriteLatency_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       46.266666412353516,
+                                                       46.599998474121094,
+                                                       46.93333435058594,
+                                                       46.86666488647461
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "virtualDisk|write_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       0.0,
+                                                       0.0,
+                                                       0.0,
+                                                       0.0
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "net|received_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       },
+                                       {
+                                               "data": [
+                                                       0.0,
+                                                       0.0,
+                                                       0.0,
+                                                       0.0
+                                               ],
+                                               "intervalUnit": {
+                                                       "quantifier": 1
+                                               },
+                                               "statKey": {
+                                                       "key": "net:vmnic5|transmitted_average"
+                                               },
+                                               "timestamps": [
+                                                       1561744712552,
+                                                       1561744772549,
+                                                       1561744832548,
+                                                       1561744892557
+                                               ]
+                                       }
+                               ]
+                       }
+               }
+       ]
+}
index b404738..c67a4b1 100644 (file)
@@ -21,3 +21,4 @@
 ##
 flake8<3.0
 mock
+requests-mock
diff --git a/tox.ini b/tox.ini
index 898e487..3a360af 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -37,6 +37,7 @@ deps = -r{toxinidir}/test-requirements.txt
 [testenv:pylint]
 basepython = python3
 deps = pylint
+       -r{toxinidir}/test-requirements.txt
 commands =
     pylint -E osm_mon