From: Benjamin Diaz Date: Mon, 4 Feb 2019 16:38:11 +0000 (-0300) Subject: Adds collection of vim status metric X-Git-Tag: v6.0.0~27 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=refs%2Fchanges%2F98%2F7198%2F2;p=osm%2FMON.git Adds collection of vim status metric Current support is only OpenStack. It tries to do a simple project list query to check if VIM is up or not. Change-Id: Ifec844d9514b6c676b065fee7cf785bdf2a2ea3d Signed-off-by: Benjamin Diaz --- diff --git a/osm_mon/cmd/mon_collector.py b/osm_mon/cmd/mon_collector.py index 6389d49..c4e2969 100644 --- a/osm_mon/cmd/mon_collector.py +++ b/osm_mon/cmd/mon_collector.py @@ -25,6 +25,7 @@ import logging import sys from osm_mon.collector.collector import Collector +from osm_mon.core.database import DatabaseManager from osm_mon.core.settings import Config @@ -45,6 +46,10 @@ def main(): log = logging.getLogger(__name__) log.info("Starting MON Collector...") log.debug("Config: %s", vars(cfg)) + log.info("Initializing database...") + db_manager = DatabaseManager() + db_manager.create_tables() + log.info("Database initialized correctly.") collector = Collector() collector.collect_forever() diff --git a/osm_mon/cmd/mon_evaluator.py b/osm_mon/cmd/mon_evaluator.py index 3343b9c..79adabd 100644 --- a/osm_mon/cmd/mon_evaluator.py +++ b/osm_mon/cmd/mon_evaluator.py @@ -24,6 +24,7 @@ import logging import sys +from osm_mon.core.database import DatabaseManager from osm_mon.core.settings import Config from osm_mon.evaluator.evaluator import Evaluator @@ -45,6 +46,10 @@ def main(): log = logging.getLogger(__name__) log.info("Starting MON Evaluator...") log.debug("Config: %s", vars(cfg)) + log.info("Initializing database...") + db_manager = DatabaseManager() + db_manager.create_tables() + log.info("Database initialized correctly.") evaluator = Evaluator() evaluator.evaluate_forever() diff --git a/osm_mon/cmd/mon_server.py b/osm_mon/cmd/mon_server.py index a100c20..34fe0b1 100644 --- a/osm_mon/cmd/mon_server.py +++ b/osm_mon/cmd/mon_server.py @@ -24,6 +24,7 @@ import logging import sys +from osm_mon.core.database import DatabaseManager from osm_mon.core.settings import Config from osm_mon.server.server import Server @@ -45,6 +46,10 @@ def main(): log = logging.getLogger(__name__) log.info("Starting MON Server...") log.debug("Config: %s", vars(cfg)) + log.info("Initializing database...") + db_manager = DatabaseManager() + db_manager.create_tables() + log.info("Database initialized correctly.") server = Server() server.run() diff --git a/osm_mon/collector/backends/prometheus.py b/osm_mon/collector/backends/prometheus.py index 31f3122..fbe6d36 100644 --- a/osm_mon/collector/backends/prometheus.py +++ b/osm_mon/collector/backends/prometheus.py @@ -49,10 +49,9 @@ class PrometheusBackend(BaseBackend): prometheus_metrics[metric.name] = GaugeMetricFamily( OSM_METRIC_PREFIX + metric.name, 'OSM metric', - labels=['ns_id', 'vnf_member_index', 'vdu_name'] + labels=list(metric.tags.keys()) ) - prometheus_metrics[metric.name].add_metric([metric.nsr_id, metric.vnf_member_index, metric.vdur_name], - metric.value) + prometheus_metrics[metric.name].add_metric(list(metric.tags.values()), metric.value) self.custom_collector.metrics = prometheus_metrics.values() def _start_exporter(self, port): diff --git a/osm_mon/collector/collector.py b/osm_mon/collector/collector.py index 3143882..36ce1b0 100644 --- a/osm_mon/collector/collector.py +++ b/osm_mon/collector/collector.py @@ -27,9 +27,11 @@ import time import peewee from osm_mon.collector.backends.prometheus import PrometheusBackend -from osm_mon.collector.collectors.juju import VCACollector -from osm_mon.collector.collectors.openstack import OpenstackCollector -from osm_mon.collector.collectors.vmware import VMwareCollector +from osm_mon.collector.vnf_collectors.vmware import VMwareCollector +from osm_mon.collector.infra_collectors.openstack import OpenstackInfraCollector +from osm_mon.collector.metric import Metric +from osm_mon.collector.vnf_collectors.juju import VCACollector +from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector from osm_mon.core.common_db import CommonDbClient from osm_mon.core.database import DatabaseManager from osm_mon.core.settings import Config @@ -40,6 +42,9 @@ VIM_COLLECTORS = { "openstack": OpenstackCollector, "vmware": VMwareCollector } +VIM_INFRA_COLLECTORS = { + "openstack": OpenstackInfraCollector +} METRIC_BACKENDS = [ PrometheusBackend ] @@ -69,7 +74,8 @@ class Collector: def _collect_vim_metrics(self, vnfr: dict, vim_account_id: str): # TODO(diazb) Add support for vrops and aws - vim_type = self.database_manager.get_vim_type(vim_account_id) + database_manager = DatabaseManager() + vim_type = database_manager.get_vim_type(vim_account_id) if vim_type in VIM_COLLECTORS: collector = VIM_COLLECTORS[vim_type](vim_account_id) metrics = collector.collect(vnfr) @@ -78,6 +84,17 @@ class Collector: else: log.debug("vimtype %s is not supported.", vim_type) + def _collect_vim_infra_metrics(self, vim_account_id: str): + database_manager = DatabaseManager() + vim_type = database_manager.get_vim_type(vim_account_id) + if vim_type in VIM_INFRA_COLLECTORS: + collector = VIM_INFRA_COLLECTORS[vim_type](vim_account_id) + status = collector.is_vim_ok() + status_metric = Metric({'vim_id': vim_account_id}, 'vim_status', status) + self.queue.put(status_metric) + else: + log.debug("vimtype %s is not supported.", vim_type) + def _collect_vca_metrics(self, vnfr: dict): log.debug('_collect_vca_metrics') log.debug('vnfr: %s', vnfr) @@ -101,6 +118,12 @@ class Collector: args=(vnfr,)) processes.append(p) p.start() + vims = self.common_db.get_vim_accounts() + for vim in vims: + p = multiprocessing.Process(target=self._collect_vim_infra_metrics, + args=(vim['_id'],)) + processes.append(p) + p.start() for process in processes: process.join() metrics = [] diff --git a/osm_mon/collector/collectors/__init__.py b/osm_mon/collector/collectors/__init__.py deleted file mode 100644 index 971f4e9..0000000 --- a/osm_mon/collector/collectors/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## diff --git a/osm_mon/collector/collectors/base.py b/osm_mon/collector/collectors/base.py deleted file mode 100644 index 824e106..0000000 --- a/osm_mon/collector/collectors/base.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## -from typing import List - -from osm_mon.collector.metric import Metric - - -class BaseCollector: - def collect(self, vnfr: dict) -> List[Metric]: - pass diff --git a/osm_mon/collector/collectors/base_vim.py b/osm_mon/collector/collectors/base_vim.py deleted file mode 100644 index 2ae0617..0000000 --- a/osm_mon/collector/collectors/base_vim.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## - -from osm_mon.collector.collectors.base import BaseCollector - - -class BaseVimCollector(BaseCollector): - def __init__(self, vim_account_id: str): - pass diff --git a/osm_mon/collector/collectors/juju.py b/osm_mon/collector/collectors/juju.py deleted file mode 100644 index 638086a..0000000 --- a/osm_mon/collector/collectors/juju.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## -import asyncio -import logging -from typing import List - -from n2vc.vnf import N2VC - -from osm_mon.collector.collectors.base import BaseCollector -from osm_mon.collector.metric import Metric -from osm_mon.core.common_db import CommonDbClient -from osm_mon.core.exceptions import VcaDeploymentInfoNotFound -from osm_mon.core.settings import Config - -log = logging.getLogger(__name__) - - -class VCACollector(BaseCollector): - def __init__(self): - cfg = Config.instance() - self.common_db = CommonDbClient() - self.loop = asyncio.get_event_loop() - self.n2vc = N2VC(server=cfg.OSMMON_VCA_HOST, user=cfg.OSMMON_VCA_USER, secret=cfg.OSMMON_VCA_SECRET) - - def collect(self, vnfr: dict) -> List[Metric]: - nsr_id = vnfr['nsr-id-ref'] - vnf_member_index = vnfr['member-vnf-index-ref'] - vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) - metrics = [] - for vdur in vnfr['vdur']: - # This avoids errors when vdur records have not been completely filled - if 'name' not in vdur: - continue - vdu = next( - filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu']) - ) - if 'vdu-configuration' in vdu and 'metrics' in vdu['vdu-configuration']: - try: - vca_deployment_info = self.get_vca_deployment_info(nsr_id, vnf_member_index, vdur['name']) - except VcaDeploymentInfoNotFound: - continue - measures = self.loop.run_until_complete(self.n2vc.GetMetrics(vca_deployment_info['model'], - vca_deployment_info['application'])) - log.debug('Measures: %s', measures) - for measure_list in measures.values(): - for measure in measure_list: - log.debug("Measure: %s", measure) - metric = Metric(nsr_id, vnf_member_index, vdur['name'], measure['key'], float(measure['value'])) - metrics.append(metric) - if 'vnf-configuration' in vnfd and 'metrics' in vnfd['vnf-configuration']: - try: - vca_deployment_info = self.get_vca_deployment_info(nsr_id, vnf_member_index, None) - except VcaDeploymentInfoNotFound: - return metrics - measures = self.loop.run_until_complete(self.n2vc.GetMetrics(vca_deployment_info['model'], - vca_deployment_info['application'])) - log.debug('Measures: %s', measures) - for measure_list in measures.values(): - for measure in measure_list: - log.debug("Measure: %s", measure) - metric = Metric(nsr_id, vnf_member_index, '', measure['key'], float(measure['value'])) - metrics.append(metric) - return metrics - - def get_vca_deployment_info(self, nsr_id, vnf_member_index, vdur_name): - nsr = self.common_db.get_nsr(nsr_id) - for vca_deployment in nsr["_admin"]["deployed"]["VCA"]: - if vca_deployment: - if vca_deployment['member-vnf-index'] == vnf_member_index and vca_deployment['vdu_name'] == vdur_name: - return vca_deployment - raise VcaDeploymentInfoNotFound("VCA deployment info for nsr_id {}, index {} and vdur_name {} not found." - .format(nsr_id, vnf_member_index, vdur_name)) diff --git a/osm_mon/collector/collectors/openstack.py b/osm_mon/collector/collectors/openstack.py deleted file mode 100644 index 30bab0c..0000000 --- a/osm_mon/collector/collectors/openstack.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## -import datetime -import json -import logging -from typing import List - -import gnocchiclient.exceptions -from gnocchiclient.v1 import client as gnocchi_client -from keystoneauth1 import session -from keystoneauth1.identity import v3 - -from osm_mon.collector.collectors.base_vim import BaseVimCollector -from osm_mon.collector.metric import Metric -from osm_mon.core.auth import AuthManager -from osm_mon.core.common_db import CommonDbClient -from osm_mon.core.settings import Config - -log = logging.getLogger(__name__) - -METRIC_MAPPINGS = { - "average_memory_utilization": "memory.usage", - "disk_read_ops": "disk.read.requests", - "disk_write_ops": "disk.write.requests", - "disk_read_bytes": "disk.read.bytes", - "disk_write_bytes": "disk.write.bytes", - "packets_dropped": "interface.if_dropped", - "packets_received": "interface.if_packets", - "packets_sent": "interface.if_packets", - "cpu_utilization": "cpu_util", -} - - -class OpenstackCollector(BaseVimCollector): - def __init__(self, vim_account_id: str): - super().__init__(vim_account_id) - self.common_db = CommonDbClient() - self.auth_manager = AuthManager() - self.granularity = self._get_granularity(vim_account_id) - self.gnocchi_client = self._build_gnocchi_client(vim_account_id) - - def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str: - vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name) - return vdur['vim-id'] - - def _build_gnocchi_client(self, vim_account_id: str) -> gnocchi_client.Client: - creds = self.auth_manager.get_credentials(vim_account_id) - verify_ssl = self.auth_manager.is_verify_ssl(vim_account_id) - auth = v3.Password(auth_url=creds.url, - username=creds.user, - password=creds.password, - project_name=creds.tenant_name, - project_domain_id='default', - user_domain_id='default') - sess = session.Session(auth=auth, verify=verify_ssl) - return gnocchi_client.Client(session=sess) - - def _get_granularity(self, vim_account_id: str): - creds = self.auth_manager.get_credentials(vim_account_id) - vim_config = json.loads(creds.config) - if 'granularity' in vim_config: - return int(vim_config['granularity']) - else: - cfg = Config.instance() - return cfg.OS_DEFAULT_GRANULARITY - - def collect(self, vnfr: dict) -> List[Metric]: - nsr_id = vnfr['nsr-id-ref'] - vnf_member_index = vnfr['member-vnf-index-ref'] - vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) - metrics = [] - for vdur in vnfr['vdur']: - # This avoids errors when vdur records have not been completely filled - if 'name' not in vdur: - continue - vdu = next( - filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu']) - ) - if 'monitoring-param' in vdu: - for param in vdu['monitoring-param']: - metric_name = param['nfvi-metric'] - gnocchi_metric_name = METRIC_MAPPINGS[metric_name] - delta = 10 * self.granularity - start_date = datetime.datetime.now() - datetime.timedelta(seconds=delta) - resource_id = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name']) - try: - measures = self.gnocchi_client.metric.get_measures(gnocchi_metric_name, - start=start_date, - resource_id=resource_id, - granularity=self.granularity) - if len(measures): - metric = Metric(nsr_id, vnf_member_index, vdur['name'], metric_name, measures[-1][2]) - metrics.append(metric) - except gnocchiclient.exceptions.NotFound as e: - log.debug("No metric found: %s", e) - pass - return metrics diff --git a/osm_mon/collector/collectors/vmware.py b/osm_mon/collector/collectors/vmware.py deleted file mode 100644 index f347446..0000000 --- a/osm_mon/collector/collectors/vmware.py +++ /dev/null @@ -1,361 +0,0 @@ -# -*- coding: utf-8 -*- - -## -# Copyright 2016-2017 VMware Inc. -# This file is part of ETSI OSM -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact: osslegalrouting@vmware.com -## - -import json -import logging - -from pyvcloud.vcd.client import BasicLoginCredentials -from pyvcloud.vcd.client import Client -import requests -import time -from xml.etree import ElementTree as XmlElementTree -import traceback -import six - -from osm_mon.collector.collectors.base_vim import BaseVimCollector -from osm_mon.collector.metric import Metric -from osm_mon.core.auth import AuthManager -from osm_mon.core.common_db import CommonDbClient -from osm_mon.core.settings import Config - -log = logging.getLogger(__name__) - -API_VERSION = '5.9' - -PERIOD_MSEC = {'HR': 3600000, - 'DAY': 86400000, - 'WEEK': 604800000, - 'MONTH': 2678400000, - 'YEAR': 31536000000} - -METRIC_MAPPINGS = { - "average_memory_utilization": "mem|usage_average", - "cpu_utilization": "cpu|usage_average", - "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average", - "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average", - "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average", - "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average", - "packets_dropped_0": "net:4000|dropped", - "packets_dropped_1": "net:4001|dropped", - "packets_dropped_2": "net:4002|dropped", - "packets_received": "net:Aggregate of all instances|packetsRxPerSec", - "packets_sent": "net:Aggregate of all instances|packetsTxPerSec", -} - -# Disable warnings from self-signed certificates. -requests.packages.urllib3.disable_warnings() - - -class VMwareCollector(BaseVimCollector): - def __init__(self, vim_account_id: str): - super().__init__(vim_account_id) - self.common_db = CommonDbClient() - self.auth_manager = AuthManager() - self.granularity = self._get_granularity(vim_account_id) - vim_account = self.get_vim_account(vim_account_id) - self.vrops_site = vim_account['vrops_site'] - self.vrops_user = vim_account['vrops_user'] - self.vrops_password = vim_account['vrops_password'] - self.vcloud_site = vim_account['vim_url'] - self.admin_username = vim_account['admin_username'] - self.admin_password = vim_account['admin_password'] - self.vim_uuid = vim_account['vim_uuid'] - - def connect_as_admin(self): - """ Method connect as pvdc admin user to vCloud director. - There are certain action that can be done only by provider vdc admin user. - Organization creation / provider network creation etc. - - Returns: - The return client object that letter can be used to connect to vcloud direct as admin for provider vdc - """ - - log.info("Logging into vCD org as admin.") - - try: - host = self.vcloud_site - admin_user = self.admin_username - admin_passwd = self.admin_password - org = 'System' - client = Client(host, verify_ssl_certs=False) - client.set_credentials(BasicLoginCredentials(admin_user, org, - admin_passwd)) - return client - - except Exception as e: - log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e)) - - def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str: - vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name) - return vdur['vim-id'] - - def get_vim_account(self, vim_account_id: str): - """ - Method to get VIM account details by its ID - arg - VIM ID - return - dict with vim account details - """ - vim_account = {} - vim_account_info = self.auth_manager.get_credentials(vim_account_id) - - vim_account['name'] = vim_account_info.name - vim_account['vim_tenant_name'] = vim_account_info.tenant_name - vim_account['vim_type'] = vim_account_info.type - vim_account['vim_url'] = vim_account_info.url - vim_account['org_user'] = vim_account_info.user - vim_account['org_password'] = vim_account_info.password - vim_account['vim_uuid'] = vim_account_info.uuid - - vim_config = json.loads(vim_account_info.config) - vim_account['admin_username'] = vim_config['admin_username'] - vim_account['admin_password'] = vim_config['admin_password'] - vim_account['vrops_site'] = vim_config['vrops_site'] - vim_account['vrops_user'] = vim_config['vrops_user'] - vim_account['vrops_password'] = vim_config['vrops_password'] - vim_account['vcenter_ip'] = vim_config['vcenter_ip'] - vim_account['vcenter_port'] = vim_config['vcenter_port'] - vim_account['vcenter_user'] = vim_config['vcenter_user'] - vim_account['vcenter_password'] = vim_config['vcenter_password'] - - if vim_config['nsx_manager'] is not None: - vim_account['nsx_manager'] = vim_config['nsx_manager'] - - if vim_config['nsx_user'] is not None: - vim_account['nsx_user'] = vim_config['nsx_user'] - - if vim_config['nsx_password'] is not None: - vim_account['nsx_password'] = vim_config['nsx_password'] - - if vim_config['orgname'] is not None: - vim_account['orgname'] = vim_config['orgname'] - - return vim_account - - def _get_granularity(self, vim_account_id: str): - creds = self.auth_manager.get_credentials(vim_account_id) - vim_config = json.loads(creds.config) - if 'granularity' in vim_config: - return int(vim_config['granularity']) - else: - cfg = Config.instance() - return cfg.OS_DEFAULT_GRANULARITY - - def get_vm_moref_id(self, vapp_uuid): - """ - Method to get the moref_id of given VM - arg - vapp_uuid - return - VM mored_id - """ - try: - if vapp_uuid: - vm_details = self.get_vapp_details_rest(vapp_uuid) - - if vm_details and "vm_vcenter_info" in vm_details: - vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None) - - log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid)) - return vm_moref_id - - except Exception as exp: - log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc())) - - def get_vapp_details_rest(self, vapp_uuid=None): - """ - Method retrieve vapp detail from vCloud director - vapp_uuid - is vapp identifier. - Returns - VM MOref ID or return None - """ - parsed_respond = {} - vca = None - - if vapp_uuid is None: - return parsed_respond - - vca = self.connect_as_admin() - - if not vca: - log.error("Failed to connect to vCD") - return parsed_respond - - url_list = [self.vcloud_site, '/api/vApp/vapp-', vapp_uuid] - get_vapp_restcall = ''.join(url_list) - - if vca._session: - headers = {'Accept': 'application/*+xml;version=' + API_VERSION, - 'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']} - response = requests.get(get_vapp_restcall, - headers=headers, - verify=False) - - if response.status_code != 200: - log.error("REST API call {} failed. Return status code {}".format(get_vapp_restcall, - response.content)) - return parsed_respond - - try: - xmlroot_respond = XmlElementTree.fromstring(response.content) - - namespaces = {'vm': 'http://www.vmware.com/vcloud/v1.5', - "vmext": "http://www.vmware.com/vcloud/extension/v1.5", - "xmlns": "http://www.vmware.com/vcloud/v1.5"} - - # parse children section for other attrib - children_section = xmlroot_respond.find('vm:Children/', namespaces) - if children_section is not None: - vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces) - if vCloud_extension_section is not None: - vm_vcenter_info = {} - vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces) - vmext = vim_info.find('vmext:VmVimObjectRef', namespaces) - if vmext is not None: - vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text - parsed_respond["vm_vcenter_info"] = vm_vcenter_info - - except Exception as exp: - log.info("Error occurred for getting vApp details: {}\n{}".format(exp, - traceback.format_exc()) - ) - - return parsed_respond - - def get_vm_resource_id(self, vm_moref_id): - """ Find resource ID in vROPs using vm_moref_id - """ - if vm_moref_id is None: - return None - - api_url = '/suite-api/api/resources?resourceKind=VirtualMachine' - headers = {'Accept': 'application/json'} - - resp = requests.get(self.vrops_site + api_url, - auth=(self.vrops_user, self.vrops_password), - verify=False, headers=headers) - - if resp.status_code != 200: - log.error("Failed to get resource details for{} {} {}".format(vm_moref_id, - resp.status_code, - resp.content)) - return None - - vm_resource_id = None - try: - resp_data = json.loads(resp.content.decode('utf-8')) - if resp_data.get('resourceList') is not None: - resource_list = resp_data.get('resourceList') - for resource in resource_list: - if resource.get('resourceKey') is not None: - resource_details = resource['resourceKey'] - if resource_details.get('resourceIdentifiers') is not None: - resource_identifiers = resource_details['resourceIdentifiers'] - for resource_identifier in resource_identifiers: - if resource_identifier['identifierType']['name'] == 'VMEntityObjectID': - if resource_identifier.get('value') is not None and \ - resource_identifier['value'] == vm_moref_id: - vm_resource_id = resource['identifier'] - log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id, - vm_moref_id)) - - except Exception as exp: - log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc())) - - return vm_resource_id - - def collect(self, vnfr: dict): - nsr_id = vnfr['nsr-id-ref'] - vnf_member_index = vnfr['member-vnf-index-ref'] - vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) - metrics = [] - for vdur in vnfr['vdur']: - # This avoids errors when vdur records have not been completely filled - if 'name' not in vdur: - continue - vdu = next( - filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu']) - - ) - if 'monitoring-param' in vdu: - for param in vdu['monitoring-param']: - metric_name = param['nfvi-metric'] - vrops_metric_name = METRIC_MAPPINGS[metric_name] - resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name']) - - # Find vm_moref_id from vApp uuid in vCD - vm_moref_id = self.get_vm_moref_id(resource_uuid) - if vm_moref_id is None: - log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid)) - return - - # Based on vm_moref_id, find VM's corresponding resource_id in vROPs - resource_id = self.get_vm_resource_id(vm_moref_id) - if resource_id is None: - log.debug("Failed to find resource in vROPs: {}".format(resource_uuid)) - return - try: - end_time = int(round(time.time() * 1000)) - time_diff = PERIOD_MSEC['YEAR'] - begin_time = end_time - time_diff - - api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format( - resource_id, vrops_metric_name, str(begin_time), str(end_time)) - - headers = {'Accept': 'application/json'} - - resp = requests.get(self.vrops_site + api_url, - auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers - ) - - if resp.status_code != 200: - log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name, - resp.status_code, - resp.content)) - return - - metrics_data = {} - m_data = json.loads(resp.content.decode('utf-8')) - - for resp_key, resp_val in six.iteritems(m_data): - if resp_key == 'values': - data = m_data['values'][0] - for data_k, data_v in six.iteritems(data): - if data_k == 'stat-list': - stat_list = data_v - for stat_list_k, stat_list_v in six.iteritems(stat_list): - for stat_keys, stat_vals in six.iteritems(stat_list_v[0]): - if stat_keys == 'timestamps': - metrics_data['time_series'] = stat_list_v[0]['timestamps'] - if stat_keys == 'data': - metrics_data['metrics_series'] = stat_list_v[0]['data'] - - if metrics_data: - metric = Metric(nsr_id, - vnf_member_index, - vdur['name'], - metric_name, - metrics_data['metrics_series'][-1]) - - metrics.append(metric) - - except Exception as e: - log.debug("No metric found: %s", e) - pass - return metrics diff --git a/osm_mon/collector/infra_collectors/__init__.py b/osm_mon/collector/infra_collectors/__init__.py new file mode 100644 index 0000000..971f4e9 --- /dev/null +++ b/osm_mon/collector/infra_collectors/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## diff --git a/osm_mon/collector/infra_collectors/base.py b/osm_mon/collector/infra_collectors/base.py new file mode 100644 index 0000000..6f137d0 --- /dev/null +++ b/osm_mon/collector/infra_collectors/base.py @@ -0,0 +1,25 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## + + +class BaseInfraCollector: + pass diff --git a/osm_mon/collector/infra_collectors/base_vim.py b/osm_mon/collector/infra_collectors/base_vim.py new file mode 100644 index 0000000..0a075a1 --- /dev/null +++ b/osm_mon/collector/infra_collectors/base_vim.py @@ -0,0 +1,30 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +from osm_mon.collector.infra_collectors.base import BaseInfraCollector + + +class BaseVimInfraCollector(BaseInfraCollector): + def __init__(self, vim_account_id: str): + pass + + def is_vim_ok(self) -> bool: + pass diff --git a/osm_mon/collector/infra_collectors/openstack.py b/osm_mon/collector/infra_collectors/openstack.py new file mode 100644 index 0000000..4237e4f --- /dev/null +++ b/osm_mon/collector/infra_collectors/openstack.py @@ -0,0 +1,58 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import logging + +from keystoneauth1 import session +from keystoneauth1.identity import v3 +from keystoneclient.v3 import client + +from osm_mon.collector.infra_collectors.base_vim import BaseVimInfraCollector +from osm_mon.core.auth import AuthManager + +log = logging.getLogger(__name__) + + +class OpenstackInfraCollector(BaseVimInfraCollector): + def __init__(self, vim_account_id: str): + super().__init__(vim_account_id) + self.auth_manager = AuthManager() + self.keystone_client = self._build_keystone_client(vim_account_id) + + def is_vim_ok(self) -> bool: + try: + self.keystone_client.projects.list() + return True + except Exception: + log.exception("VIM status is not OK!") + return False + + def _build_keystone_client(self, vim_account_id): + creds = self.auth_manager.get_credentials(vim_account_id) + verify_ssl = self.auth_manager.is_verify_ssl(vim_account_id) + auth = v3.Password(auth_url=creds.url, + username=creds.user, + password=creds.password, + project_name=creds.tenant_name, + project_domain_id='default', + user_domain_id='default') + sess = session.Session(auth=auth, verify=verify_ssl) + return client.Client(session=sess) diff --git a/osm_mon/collector/metric.py b/osm_mon/collector/metric.py index e8da11b..741096f 100644 --- a/osm_mon/collector/metric.py +++ b/osm_mon/collector/metric.py @@ -22,9 +22,7 @@ class Metric: - def __init__(self, nsr_id, vnf_member_index, vdur_name, name, value): - self.nsr_id = nsr_id - self.vnf_member_index = vnf_member_index - self.vdur_name = vdur_name + def __init__(self, tags: dict, name: str, value): + self.tags = tags self.name = name self.value = value diff --git a/osm_mon/collector/vnf_collectors/__init__.py b/osm_mon/collector/vnf_collectors/__init__.py new file mode 100644 index 0000000..971f4e9 --- /dev/null +++ b/osm_mon/collector/vnf_collectors/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## diff --git a/osm_mon/collector/vnf_collectors/base.py b/osm_mon/collector/vnf_collectors/base.py new file mode 100644 index 0000000..824e106 --- /dev/null +++ b/osm_mon/collector/vnf_collectors/base.py @@ -0,0 +1,29 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +from typing import List + +from osm_mon.collector.metric import Metric + + +class BaseCollector: + def collect(self, vnfr: dict) -> List[Metric]: + pass diff --git a/osm_mon/collector/vnf_collectors/base_vim.py b/osm_mon/collector/vnf_collectors/base_vim.py new file mode 100644 index 0000000..29a348f --- /dev/null +++ b/osm_mon/collector/vnf_collectors/base_vim.py @@ -0,0 +1,28 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## + +from osm_mon.collector.vnf_collectors.base import BaseCollector + + +class BaseVimCollector(BaseCollector): + def __init__(self, vim_account_id: str): + pass diff --git a/osm_mon/collector/vnf_collectors/juju.py b/osm_mon/collector/vnf_collectors/juju.py new file mode 100644 index 0000000..928b35a --- /dev/null +++ b/osm_mon/collector/vnf_collectors/juju.py @@ -0,0 +1,93 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import asyncio +import logging +from typing import List + +from n2vc.vnf import N2VC + +from osm_mon.collector.metric import Metric +from osm_mon.collector.vnf_collectors.base import BaseCollector +from osm_mon.collector.vnf_metric import VnfMetric +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.exceptions import VcaDeploymentInfoNotFound +from osm_mon.core.settings import Config + +log = logging.getLogger(__name__) + + +class VCACollector(BaseCollector): + def __init__(self): + cfg = Config.instance() + self.common_db = CommonDbClient() + self.loop = asyncio.get_event_loop() + self.n2vc = N2VC(server=cfg.OSMMON_VCA_HOST, user=cfg.OSMMON_VCA_USER, secret=cfg.OSMMON_VCA_SECRET) + + def collect(self, vnfr: dict) -> List[Metric]: + nsr_id = vnfr['nsr-id-ref'] + vnf_member_index = vnfr['member-vnf-index-ref'] + vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) + metrics = [] + for vdur in vnfr['vdur']: + # This avoids errors when vdur records have not been completely filled + if 'name' not in vdur: + continue + vdu = next( + filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu']) + ) + if 'vdu-configuration' in vdu and 'metrics' in vdu['vdu-configuration']: + try: + vca_deployment_info = self.get_vca_deployment_info(nsr_id, vnf_member_index, vdur['name']) + except VcaDeploymentInfoNotFound: + continue + measures = self.loop.run_until_complete(self.n2vc.GetMetrics(vca_deployment_info['model'], + vca_deployment_info['application'])) + log.debug('Measures: %s', measures) + for measure_list in measures.values(): + for measure in measure_list: + log.debug("Measure: %s", measure) + metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], measure['key'], + float(measure['value'])) + metrics.append(metric) + if 'vnf-configuration' in vnfd and 'metrics' in vnfd['vnf-configuration']: + try: + vca_deployment_info = self.get_vca_deployment_info(nsr_id, vnf_member_index, None) + except VcaDeploymentInfoNotFound: + return metrics + measures = self.loop.run_until_complete(self.n2vc.GetMetrics(vca_deployment_info['model'], + vca_deployment_info['application'])) + log.debug('Measures: %s', measures) + for measure_list in measures.values(): + for measure in measure_list: + log.debug("Measure: %s", measure) + metric = VnfMetric(nsr_id, vnf_member_index, '', measure['key'], float(measure['value'])) + metrics.append(metric) + return metrics + + def get_vca_deployment_info(self, nsr_id, vnf_member_index, vdur_name): + nsr = self.common_db.get_nsr(nsr_id) + for vca_deployment in nsr["_admin"]["deployed"]["VCA"]: + if vca_deployment: + if vca_deployment['member-vnf-index'] == vnf_member_index and vca_deployment['vdu_name'] == vdur_name: + return vca_deployment + raise VcaDeploymentInfoNotFound("VCA deployment info for nsr_id {}, index {} and vdur_name {} not found." + .format(nsr_id, vnf_member_index, vdur_name)) diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py new file mode 100644 index 0000000..8dbab5c --- /dev/null +++ b/osm_mon/collector/vnf_collectors/openstack.py @@ -0,0 +1,117 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import datetime +import json +import logging +from typing import List + +import gnocchiclient.exceptions +from gnocchiclient.v1 import client as gnocchi_client +from keystoneauth1 import session +from keystoneauth1.identity import v3 + +from osm_mon.collector.metric import Metric +from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector +from osm_mon.collector.vnf_metric import VnfMetric +from osm_mon.core.auth import AuthManager +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.settings import Config + +log = logging.getLogger(__name__) + +METRIC_MAPPINGS = { + "average_memory_utilization": "memory.usage", + "disk_read_ops": "disk.read.requests", + "disk_write_ops": "disk.write.requests", + "disk_read_bytes": "disk.read.bytes", + "disk_write_bytes": "disk.write.bytes", + "packets_dropped": "interface.if_dropped", + "packets_received": "interface.if_packets", + "packets_sent": "interface.if_packets", + "cpu_utilization": "cpu_util", +} + + +class OpenstackCollector(BaseVimCollector): + def __init__(self, vim_account_id: str): + super().__init__(vim_account_id) + self.common_db = CommonDbClient() + self.auth_manager = AuthManager() + self.granularity = self._get_granularity(vim_account_id) + self.gnocchi_client = self._build_gnocchi_client(vim_account_id) + + def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str: + vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name) + return vdur['vim-id'] + + def _build_gnocchi_client(self, vim_account_id: str) -> gnocchi_client.Client: + creds = self.auth_manager.get_credentials(vim_account_id) + verify_ssl = self.auth_manager.is_verify_ssl(vim_account_id) + auth = v3.Password(auth_url=creds.url, + username=creds.user, + password=creds.password, + project_name=creds.tenant_name, + project_domain_id='default', + user_domain_id='default') + sess = session.Session(auth=auth, verify=verify_ssl) + return gnocchi_client.Client(session=sess) + + def _get_granularity(self, vim_account_id: str): + creds = self.auth_manager.get_credentials(vim_account_id) + vim_config = json.loads(creds.config) + if 'granularity' in vim_config: + return int(vim_config['granularity']) + else: + cfg = Config.instance() + return cfg.OS_DEFAULT_GRANULARITY + + def collect(self, vnfr: dict) -> List[Metric]: + nsr_id = vnfr['nsr-id-ref'] + vnf_member_index = vnfr['member-vnf-index-ref'] + vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) + metrics = [] + for vdur in vnfr['vdur']: + # This avoids errors when vdur records have not been completely filled + if 'name' not in vdur: + continue + vdu = next( + filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu']) + ) + if 'monitoring-param' in vdu: + for param in vdu['monitoring-param']: + metric_name = param['nfvi-metric'] + gnocchi_metric_name = METRIC_MAPPINGS[metric_name] + delta = 10 * self.granularity + start_date = datetime.datetime.now() - datetime.timedelta(seconds=delta) + resource_id = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name']) + try: + measures = self.gnocchi_client.metric.get_measures(gnocchi_metric_name, + start=start_date, + resource_id=resource_id, + granularity=self.granularity) + if len(measures): + metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], metric_name, measures[-1][2]) + metrics.append(metric) + except gnocchiclient.exceptions.NotFound as e: + log.debug("No metric found: %s", e) + pass + return metrics diff --git a/osm_mon/collector/vnf_collectors/vmware.py b/osm_mon/collector/vnf_collectors/vmware.py new file mode 100644 index 0000000..7402afb --- /dev/null +++ b/osm_mon/collector/vnf_collectors/vmware.py @@ -0,0 +1,361 @@ +# -*- coding: utf-8 -*- + +## +# Copyright 2016-2017 VMware Inc. +# This file is part of ETSI OSM +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact: osslegalrouting@vmware.com +## + +import json +import logging +import time +import traceback +from xml.etree import ElementTree as XmlElementTree + +import requests +import six +from pyvcloud.vcd.client import BasicLoginCredentials +from pyvcloud.vcd.client import Client + +from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector +from osm_mon.collector.vnf_metric import VnfMetric +from osm_mon.core.auth import AuthManager +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.settings import Config + +log = logging.getLogger(__name__) + +API_VERSION = '5.9' + +PERIOD_MSEC = {'HR': 3600000, + 'DAY': 86400000, + 'WEEK': 604800000, + 'MONTH': 2678400000, + 'YEAR': 31536000000} + +METRIC_MAPPINGS = { + "average_memory_utilization": "mem|usage_average", + "cpu_utilization": "cpu|usage_average", + "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average", + "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average", + "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average", + "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average", + "packets_dropped_0": "net:4000|dropped", + "packets_dropped_1": "net:4001|dropped", + "packets_dropped_2": "net:4002|dropped", + "packets_received": "net:Aggregate of all instances|packetsRxPerSec", + "packets_sent": "net:Aggregate of all instances|packetsTxPerSec", +} + +# Disable warnings from self-signed certificates. +requests.packages.urllib3.disable_warnings() + + +class VMwareCollector(BaseVimCollector): + def __init__(self, vim_account_id: str): + super().__init__(vim_account_id) + self.common_db = CommonDbClient() + self.auth_manager = AuthManager() + self.granularity = self._get_granularity(vim_account_id) + vim_account = self.get_vim_account(vim_account_id) + self.vrops_site = vim_account['vrops_site'] + self.vrops_user = vim_account['vrops_user'] + self.vrops_password = vim_account['vrops_password'] + self.vcloud_site = vim_account['vim_url'] + self.admin_username = vim_account['admin_username'] + self.admin_password = vim_account['admin_password'] + self.vim_uuid = vim_account['vim_uuid'] + + def connect_as_admin(self): + """ Method connect as pvdc admin user to vCloud director. + There are certain action that can be done only by provider vdc admin user. + Organization creation / provider network creation etc. + + Returns: + The return client object that letter can be used to connect to vcloud direct as admin for provider vdc + """ + + log.info("Logging into vCD org as admin.") + + try: + host = self.vcloud_site + admin_user = self.admin_username + admin_passwd = self.admin_password + org = 'System' + client = Client(host, verify_ssl_certs=False) + client.set_credentials(BasicLoginCredentials(admin_user, org, + admin_passwd)) + return client + + except Exception as e: + log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e)) + + def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str: + vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name) + return vdur['vim-id'] + + def get_vim_account(self, vim_account_id: str): + """ + Method to get VIM account details by its ID + arg - VIM ID + return - dict with vim account details + """ + vim_account = {} + vim_account_info = self.auth_manager.get_credentials(vim_account_id) + + vim_account['name'] = vim_account_info.name + vim_account['vim_tenant_name'] = vim_account_info.tenant_name + vim_account['vim_type'] = vim_account_info.type + vim_account['vim_url'] = vim_account_info.url + vim_account['org_user'] = vim_account_info.user + vim_account['org_password'] = vim_account_info.password + vim_account['vim_uuid'] = vim_account_info.uuid + + vim_config = json.loads(vim_account_info.config) + vim_account['admin_username'] = vim_config['admin_username'] + vim_account['admin_password'] = vim_config['admin_password'] + vim_account['vrops_site'] = vim_config['vrops_site'] + vim_account['vrops_user'] = vim_config['vrops_user'] + vim_account['vrops_password'] = vim_config['vrops_password'] + vim_account['vcenter_ip'] = vim_config['vcenter_ip'] + vim_account['vcenter_port'] = vim_config['vcenter_port'] + vim_account['vcenter_user'] = vim_config['vcenter_user'] + vim_account['vcenter_password'] = vim_config['vcenter_password'] + + if vim_config['nsx_manager'] is not None: + vim_account['nsx_manager'] = vim_config['nsx_manager'] + + if vim_config['nsx_user'] is not None: + vim_account['nsx_user'] = vim_config['nsx_user'] + + if vim_config['nsx_password'] is not None: + vim_account['nsx_password'] = vim_config['nsx_password'] + + if vim_config['orgname'] is not None: + vim_account['orgname'] = vim_config['orgname'] + + return vim_account + + def _get_granularity(self, vim_account_id: str): + creds = self.auth_manager.get_credentials(vim_account_id) + vim_config = json.loads(creds.config) + if 'granularity' in vim_config: + return int(vim_config['granularity']) + else: + cfg = Config.instance() + return cfg.OS_DEFAULT_GRANULARITY + + def get_vm_moref_id(self, vapp_uuid): + """ + Method to get the moref_id of given VM + arg - vapp_uuid + return - VM mored_id + """ + try: + if vapp_uuid: + vm_details = self.get_vapp_details_rest(vapp_uuid) + + if vm_details and "vm_vcenter_info" in vm_details: + vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None) + + log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid)) + return vm_moref_id + + except Exception as exp: + log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc())) + + def get_vapp_details_rest(self, vapp_uuid=None): + """ + Method retrieve vapp detail from vCloud director + vapp_uuid - is vapp identifier. + Returns - VM MOref ID or return None + """ + parsed_respond = {} + vca = None + + if vapp_uuid is None: + return parsed_respond + + vca = self.connect_as_admin() + + if not vca: + log.error("Failed to connect to vCD") + return parsed_respond + + url_list = [self.vcloud_site, '/api/vApp/vapp-', vapp_uuid] + get_vapp_restcall = ''.join(url_list) + + if vca._session: + headers = {'Accept': 'application/*+xml;version=' + API_VERSION, + 'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']} + response = requests.get(get_vapp_restcall, + headers=headers, + verify=False) + + if response.status_code != 200: + log.error("REST API call {} failed. Return status code {}".format(get_vapp_restcall, + response.content)) + return parsed_respond + + try: + xmlroot_respond = XmlElementTree.fromstring(response.content) + + namespaces = {'vm': 'http://www.vmware.com/vcloud/v1.5', + "vmext": "http://www.vmware.com/vcloud/extension/v1.5", + "xmlns": "http://www.vmware.com/vcloud/v1.5"} + + # parse children section for other attrib + children_section = xmlroot_respond.find('vm:Children/', namespaces) + if children_section is not None: + vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces) + if vCloud_extension_section is not None: + vm_vcenter_info = {} + vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces) + vmext = vim_info.find('vmext:VmVimObjectRef', namespaces) + if vmext is not None: + vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text + parsed_respond["vm_vcenter_info"] = vm_vcenter_info + + except Exception as exp: + log.info("Error occurred for getting vApp details: {}\n{}".format(exp, + traceback.format_exc()) + ) + + return parsed_respond + + def get_vm_resource_id(self, vm_moref_id): + """ Find resource ID in vROPs using vm_moref_id + """ + if vm_moref_id is None: + return None + + api_url = '/suite-api/api/resources?resourceKind=VirtualMachine' + headers = {'Accept': 'application/json'} + + resp = requests.get(self.vrops_site + api_url, + auth=(self.vrops_user, self.vrops_password), + verify=False, headers=headers) + + if resp.status_code != 200: + log.error("Failed to get resource details for{} {} {}".format(vm_moref_id, + resp.status_code, + resp.content)) + return None + + vm_resource_id = None + try: + resp_data = json.loads(resp.content.decode('utf-8')) + if resp_data.get('resourceList') is not None: + resource_list = resp_data.get('resourceList') + for resource in resource_list: + if resource.get('resourceKey') is not None: + resource_details = resource['resourceKey'] + if resource_details.get('resourceIdentifiers') is not None: + resource_identifiers = resource_details['resourceIdentifiers'] + for resource_identifier in resource_identifiers: + if resource_identifier['identifierType']['name'] == 'VMEntityObjectID': + if resource_identifier.get('value') is not None and \ + resource_identifier['value'] == vm_moref_id: + vm_resource_id = resource['identifier'] + log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id, + vm_moref_id)) + + except Exception as exp: + log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc())) + + return vm_resource_id + + def collect(self, vnfr: dict): + nsr_id = vnfr['nsr-id-ref'] + vnf_member_index = vnfr['member-vnf-index-ref'] + vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) + metrics = [] + for vdur in vnfr['vdur']: + # This avoids errors when vdur records have not been completely filled + if 'name' not in vdur: + continue + vdu = next( + filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu']) + + ) + if 'monitoring-param' in vdu: + for param in vdu['monitoring-param']: + metric_name = param['nfvi-metric'] + vrops_metric_name = METRIC_MAPPINGS[metric_name] + resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name']) + + # Find vm_moref_id from vApp uuid in vCD + vm_moref_id = self.get_vm_moref_id(resource_uuid) + if vm_moref_id is None: + log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid)) + return + + # Based on vm_moref_id, find VM's corresponding resource_id in vROPs + resource_id = self.get_vm_resource_id(vm_moref_id) + if resource_id is None: + log.debug("Failed to find resource in vROPs: {}".format(resource_uuid)) + return + try: + end_time = int(round(time.time() * 1000)) + time_diff = PERIOD_MSEC['YEAR'] + begin_time = end_time - time_diff + + api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format( + resource_id, vrops_metric_name, str(begin_time), str(end_time)) + + headers = {'Accept': 'application/json'} + + resp = requests.get(self.vrops_site + api_url, + auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers + ) + + if resp.status_code != 200: + log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name, + resp.status_code, + resp.content)) + return + + metrics_data = {} + m_data = json.loads(resp.content.decode('utf-8')) + + for resp_key, resp_val in six.iteritems(m_data): + if resp_key == 'values': + data = m_data['values'][0] + for data_k, data_v in six.iteritems(data): + if data_k == 'stat-list': + stat_list = data_v + for stat_list_k, stat_list_v in six.iteritems(stat_list): + for stat_keys, stat_vals in six.iteritems(stat_list_v[0]): + if stat_keys == 'timestamps': + metrics_data['time_series'] = stat_list_v[0]['timestamps'] + if stat_keys == 'data': + metrics_data['metrics_series'] = stat_list_v[0]['data'] + + if metrics_data: + metric = VnfMetric(nsr_id, + vnf_member_index, + vdur['name'], + metric_name, + metrics_data['metrics_series'][-1]) + + metrics.append(metric) + + except Exception as e: + log.debug("No metric found: %s", e) + pass + return metrics diff --git a/osm_mon/collector/vnf_metric.py b/osm_mon/collector/vnf_metric.py new file mode 100644 index 0000000..d0f39c3 --- /dev/null +++ b/osm_mon/collector/vnf_metric.py @@ -0,0 +1,32 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +from osm_mon.collector.metric import Metric + + +class VnfMetric(Metric): + def __init__(self, nsr_id, vnf_member_index, vdur_name, name, value): + tags = { + 'ns_id': nsr_id, + 'vnf_member_index': vnf_member_index, + 'vdu_name': vdur_name + } + super().__init__(tags, name, value) diff --git a/osm_mon/core/common_db.py b/osm_mon/core/common_db.py index beaac3d..9cc9c06 100644 --- a/osm_mon/core/common_db.py +++ b/osm_mon/core/common_db.py @@ -75,3 +75,6 @@ class CommonDbClient: def get_vim_account_id(self, nsr_id: str, vnf_member_index: int) -> str: vnfr = self.get_vnfr(nsr_id, vnf_member_index) return vnfr['vim-account-id'] + + def get_vim_accounts(self): + return self.common_db.get_list('vim_accounts') diff --git a/osm_mon/core/database.py b/osm_mon/core/database.py index a41c0fd..4cbd75f 100644 --- a/osm_mon/core/database.py +++ b/osm_mon/core/database.py @@ -25,16 +25,14 @@ import logging import uuid -from peewee import CharField, TextField, FloatField, Model, AutoField +from peewee import CharField, TextField, FloatField, Model, AutoField, Proxy from playhouse.db_url import connect from osm_mon.core.settings import Config log = logging.getLogger(__name__) -cfg = Config.instance() -cfg.read_environ() -db = connect(cfg.DATABASE) +db = Proxy() class BaseModel(Model): @@ -69,54 +67,60 @@ class Alarm(BaseModel): class DatabaseManager: + def __init__(self): + cfg = Config.instance() + cfg.read_environ() + db.initialize(connect(cfg.DATABASE)) + def create_tables(self) -> None: - try: - db.connect() + with db.atomic(): db.create_tables([VimCredentials, Alarm]) - db.close() - except Exception: - log.exception("Error creating tables: ") - def get_credentials(self, vim_uuid) -> VimCredentials: - return VimCredentials.get_or_none(VimCredentials.uuid == vim_uuid) + def get_credentials(self, vim_uuid: str = None) -> VimCredentials: + with db.atomic(): + return VimCredentials.get_or_none(VimCredentials.uuid == vim_uuid) def save_credentials(self, vim_credentials) -> VimCredentials: """Saves vim credentials. If a record with same uuid exists, overwrite it.""" - exists = VimCredentials.get_or_none(VimCredentials.uuid == vim_credentials.uuid) - if exists: - vim_credentials.id = exists.id - vim_credentials.save() - return vim_credentials + with db.atomic(): + exists = VimCredentials.get_or_none(VimCredentials.uuid == vim_credentials.uuid) + if exists: + vim_credentials.id = exists.id + vim_credentials.save() + return vim_credentials def get_alarm(self, alarm_id) -> Alarm: - alarm = (Alarm.select() - .where(Alarm.alarm_id == alarm_id) - .get()) - return alarm + with db.atomic(): + alarm = (Alarm.select() + .where(Alarm.alarm_id == alarm_id) + .get()) + return alarm def save_alarm(self, name, threshold, operation, severity, statistic, metric_name, vdur_name, vnf_member_index, nsr_id) -> Alarm: """Saves alarm.""" # TODO: Add uuid optional param and check if exists to handle updates (see self.save_credentials) - alarm = Alarm() - alarm.uuid = str(uuid.uuid4()) - alarm.name = name - alarm.threshold = threshold - alarm.operation = operation - alarm.severity = severity - alarm.statistic = statistic - alarm.monitoring_param = metric_name - alarm.vdur_name = vdur_name - alarm.vnf_member_index = vnf_member_index - alarm.nsr_id = nsr_id - alarm.save() - return alarm + with db.atomic(): + alarm = Alarm() + alarm.uuid = str(uuid.uuid4()) + alarm.name = name + alarm.threshold = threshold + alarm.operation = operation + alarm.severity = severity + alarm.statistic = statistic + alarm.monitoring_param = metric_name + alarm.vdur_name = vdur_name + alarm.vnf_member_index = vnf_member_index + alarm.nsr_id = nsr_id + alarm.save() + return alarm def delete_alarm(self, alarm_uuid) -> None: - alarm = (Alarm.select() - .where(Alarm.uuid == alarm_uuid) - .get()) - alarm.delete_instance() + with db.atomic(): + alarm = (Alarm.select() + .where(Alarm.uuid == alarm_uuid) + .get()) + alarm.delete_instance() def get_vim_type(self, vim_account_id) -> str: """Get the vim type that is required by the message.""" diff --git a/osm_mon/tests/collector/test_collector.py b/osm_mon/tests/collector/test_collector.py index 9159f35..c386ed2 100644 --- a/osm_mon/tests/collector/test_collector.py +++ b/osm_mon/tests/collector/test_collector.py @@ -25,7 +25,7 @@ import unittest from unittest import mock from osm_mon.collector.collector import Collector -from osm_mon.collector.collectors.openstack import OpenstackCollector +from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector from osm_mon.core.database import DatabaseManager, db