import sys
from osm_mon.collector.collector import Collector
+from osm_mon.core.database import DatabaseManager
from osm_mon.core.settings import Config
log = logging.getLogger(__name__)
log.info("Starting MON Collector...")
log.debug("Config: %s", vars(cfg))
+ log.info("Initializing database...")
+ db_manager = DatabaseManager()
+ db_manager.create_tables()
+ log.info("Database initialized correctly.")
collector = Collector()
collector.collect_forever()
import logging
import sys
+from osm_mon.core.database import DatabaseManager
from osm_mon.core.settings import Config
from osm_mon.evaluator.evaluator import Evaluator
log = logging.getLogger(__name__)
log.info("Starting MON Evaluator...")
log.debug("Config: %s", vars(cfg))
+ log.info("Initializing database...")
+ db_manager = DatabaseManager()
+ db_manager.create_tables()
+ log.info("Database initialized correctly.")
evaluator = Evaluator()
evaluator.evaluate_forever()
import logging
import sys
+from osm_mon.core.database import DatabaseManager
from osm_mon.core.settings import Config
from osm_mon.server.server import Server
log = logging.getLogger(__name__)
log.info("Starting MON Server...")
log.debug("Config: %s", vars(cfg))
+ log.info("Initializing database...")
+ db_manager = DatabaseManager()
+ db_manager.create_tables()
+ log.info("Database initialized correctly.")
server = Server()
server.run()
prometheus_metrics[metric.name] = GaugeMetricFamily(
OSM_METRIC_PREFIX + metric.name,
'OSM metric',
- labels=['ns_id', 'vnf_member_index', 'vdu_name']
+ labels=list(metric.tags.keys())
)
- prometheus_metrics[metric.name].add_metric([metric.nsr_id, metric.vnf_member_index, metric.vdur_name],
- metric.value)
+ prometheus_metrics[metric.name].add_metric(list(metric.tags.values()), metric.value)
self.custom_collector.metrics = prometheus_metrics.values()
def _start_exporter(self, port):
import peewee
from osm_mon.collector.backends.prometheus import PrometheusBackend
-from osm_mon.collector.collectors.juju import VCACollector
-from osm_mon.collector.collectors.openstack import OpenstackCollector
-from osm_mon.collector.collectors.vmware import VMwareCollector
+from osm_mon.collector.vnf_collectors.vmware import VMwareCollector
+from osm_mon.collector.infra_collectors.openstack import OpenstackInfraCollector
+from osm_mon.collector.metric import Metric
+from osm_mon.collector.vnf_collectors.juju import VCACollector
+from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector
from osm_mon.core.common_db import CommonDbClient
from osm_mon.core.database import DatabaseManager
from osm_mon.core.settings import Config
"openstack": OpenstackCollector,
"vmware": VMwareCollector
}
+VIM_INFRA_COLLECTORS = {
+ "openstack": OpenstackInfraCollector
+}
METRIC_BACKENDS = [
PrometheusBackend
]
def _collect_vim_metrics(self, vnfr: dict, vim_account_id: str):
# TODO(diazb) Add support for vrops and aws
- vim_type = self.database_manager.get_vim_type(vim_account_id)
+ database_manager = DatabaseManager()
+ vim_type = database_manager.get_vim_type(vim_account_id)
if vim_type in VIM_COLLECTORS:
collector = VIM_COLLECTORS[vim_type](vim_account_id)
metrics = collector.collect(vnfr)
else:
log.debug("vimtype %s is not supported.", vim_type)
+ def _collect_vim_infra_metrics(self, vim_account_id: str):
+ database_manager = DatabaseManager()
+ vim_type = database_manager.get_vim_type(vim_account_id)
+ if vim_type in VIM_INFRA_COLLECTORS:
+ collector = VIM_INFRA_COLLECTORS[vim_type](vim_account_id)
+ status = collector.is_vim_ok()
+ status_metric = Metric({'vim_id': vim_account_id}, 'vim_status', status)
+ self.queue.put(status_metric)
+ else:
+ log.debug("vimtype %s is not supported.", vim_type)
+
def _collect_vca_metrics(self, vnfr: dict):
log.debug('_collect_vca_metrics')
log.debug('vnfr: %s', vnfr)
args=(vnfr,))
processes.append(p)
p.start()
+ vims = self.common_db.get_vim_accounts()
+ for vim in vims:
+ p = multiprocessing.Process(target=self._collect_vim_infra_metrics,
+ args=(vim['_id'],))
+ processes.append(p)
+ p.start()
for process in processes:
process.join()
metrics = []
+++ /dev/null
-# Copyright 2018 Whitestack, LLC
-# *************************************************************
-
-# This file is part of OSM Monitoring module
-# All Rights Reserved to Whitestack, LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-
-# http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact: bdiaz@whitestack.com or glavado@whitestack.com
-##
+++ /dev/null
-# Copyright 2018 Whitestack, LLC
-# *************************************************************
-
-# This file is part of OSM Monitoring module
-# All Rights Reserved to Whitestack, LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-
-# http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact: bdiaz@whitestack.com or glavado@whitestack.com
-##
-from typing import List
-
-from osm_mon.collector.metric import Metric
-
-
-class BaseCollector:
- def collect(self, vnfr: dict) -> List[Metric]:
- pass
+++ /dev/null
-# Copyright 2018 Whitestack, LLC
-# *************************************************************
-
-# This file is part of OSM Monitoring module
-# All Rights Reserved to Whitestack, LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-
-# http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact: bdiaz@whitestack.com or glavado@whitestack.com
-##
-
-from osm_mon.collector.collectors.base import BaseCollector
-
-
-class BaseVimCollector(BaseCollector):
- def __init__(self, vim_account_id: str):
- pass
+++ /dev/null
-# Copyright 2018 Whitestack, LLC
-# *************************************************************
-
-# This file is part of OSM Monitoring module
-# All Rights Reserved to Whitestack, LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-
-# http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact: bdiaz@whitestack.com or glavado@whitestack.com
-##
-import asyncio
-import logging
-from typing import List
-
-from n2vc.vnf import N2VC
-
-from osm_mon.collector.collectors.base import BaseCollector
-from osm_mon.collector.metric import Metric
-from osm_mon.core.common_db import CommonDbClient
-from osm_mon.core.exceptions import VcaDeploymentInfoNotFound
-from osm_mon.core.settings import Config
-
-log = logging.getLogger(__name__)
-
-
-class VCACollector(BaseCollector):
- def __init__(self):
- cfg = Config.instance()
- self.common_db = CommonDbClient()
- self.loop = asyncio.get_event_loop()
- self.n2vc = N2VC(server=cfg.OSMMON_VCA_HOST, user=cfg.OSMMON_VCA_USER, secret=cfg.OSMMON_VCA_SECRET)
-
- def collect(self, vnfr: dict) -> List[Metric]:
- nsr_id = vnfr['nsr-id-ref']
- vnf_member_index = vnfr['member-vnf-index-ref']
- vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
- metrics = []
- for vdur in vnfr['vdur']:
- # This avoids errors when vdur records have not been completely filled
- if 'name' not in vdur:
- continue
- vdu = next(
- filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
- )
- if 'vdu-configuration' in vdu and 'metrics' in vdu['vdu-configuration']:
- try:
- vca_deployment_info = self.get_vca_deployment_info(nsr_id, vnf_member_index, vdur['name'])
- except VcaDeploymentInfoNotFound:
- continue
- measures = self.loop.run_until_complete(self.n2vc.GetMetrics(vca_deployment_info['model'],
- vca_deployment_info['application']))
- log.debug('Measures: %s', measures)
- for measure_list in measures.values():
- for measure in measure_list:
- log.debug("Measure: %s", measure)
- metric = Metric(nsr_id, vnf_member_index, vdur['name'], measure['key'], float(measure['value']))
- metrics.append(metric)
- if 'vnf-configuration' in vnfd and 'metrics' in vnfd['vnf-configuration']:
- try:
- vca_deployment_info = self.get_vca_deployment_info(nsr_id, vnf_member_index, None)
- except VcaDeploymentInfoNotFound:
- return metrics
- measures = self.loop.run_until_complete(self.n2vc.GetMetrics(vca_deployment_info['model'],
- vca_deployment_info['application']))
- log.debug('Measures: %s', measures)
- for measure_list in measures.values():
- for measure in measure_list:
- log.debug("Measure: %s", measure)
- metric = Metric(nsr_id, vnf_member_index, '', measure['key'], float(measure['value']))
- metrics.append(metric)
- return metrics
-
- def get_vca_deployment_info(self, nsr_id, vnf_member_index, vdur_name):
- nsr = self.common_db.get_nsr(nsr_id)
- for vca_deployment in nsr["_admin"]["deployed"]["VCA"]:
- if vca_deployment:
- if vca_deployment['member-vnf-index'] == vnf_member_index and vca_deployment['vdu_name'] == vdur_name:
- return vca_deployment
- raise VcaDeploymentInfoNotFound("VCA deployment info for nsr_id {}, index {} and vdur_name {} not found."
- .format(nsr_id, vnf_member_index, vdur_name))
+++ /dev/null
-# Copyright 2018 Whitestack, LLC
-# *************************************************************
-
-# This file is part of OSM Monitoring module
-# All Rights Reserved to Whitestack, LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-
-# http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact: bdiaz@whitestack.com or glavado@whitestack.com
-##
-import datetime
-import json
-import logging
-from typing import List
-
-import gnocchiclient.exceptions
-from gnocchiclient.v1 import client as gnocchi_client
-from keystoneauth1 import session
-from keystoneauth1.identity import v3
-
-from osm_mon.collector.collectors.base_vim import BaseVimCollector
-from osm_mon.collector.metric import Metric
-from osm_mon.core.auth import AuthManager
-from osm_mon.core.common_db import CommonDbClient
-from osm_mon.core.settings import Config
-
-log = logging.getLogger(__name__)
-
-METRIC_MAPPINGS = {
- "average_memory_utilization": "memory.usage",
- "disk_read_ops": "disk.read.requests",
- "disk_write_ops": "disk.write.requests",
- "disk_read_bytes": "disk.read.bytes",
- "disk_write_bytes": "disk.write.bytes",
- "packets_dropped": "interface.if_dropped",
- "packets_received": "interface.if_packets",
- "packets_sent": "interface.if_packets",
- "cpu_utilization": "cpu_util",
-}
-
-
-class OpenstackCollector(BaseVimCollector):
- def __init__(self, vim_account_id: str):
- super().__init__(vim_account_id)
- self.common_db = CommonDbClient()
- self.auth_manager = AuthManager()
- self.granularity = self._get_granularity(vim_account_id)
- self.gnocchi_client = self._build_gnocchi_client(vim_account_id)
-
- def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
- vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
- return vdur['vim-id']
-
- def _build_gnocchi_client(self, vim_account_id: str) -> gnocchi_client.Client:
- creds = self.auth_manager.get_credentials(vim_account_id)
- verify_ssl = self.auth_manager.is_verify_ssl(vim_account_id)
- auth = v3.Password(auth_url=creds.url,
- username=creds.user,
- password=creds.password,
- project_name=creds.tenant_name,
- project_domain_id='default',
- user_domain_id='default')
- sess = session.Session(auth=auth, verify=verify_ssl)
- return gnocchi_client.Client(session=sess)
-
- def _get_granularity(self, vim_account_id: str):
- creds = self.auth_manager.get_credentials(vim_account_id)
- vim_config = json.loads(creds.config)
- if 'granularity' in vim_config:
- return int(vim_config['granularity'])
- else:
- cfg = Config.instance()
- return cfg.OS_DEFAULT_GRANULARITY
-
- def collect(self, vnfr: dict) -> List[Metric]:
- nsr_id = vnfr['nsr-id-ref']
- vnf_member_index = vnfr['member-vnf-index-ref']
- vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
- metrics = []
- for vdur in vnfr['vdur']:
- # This avoids errors when vdur records have not been completely filled
- if 'name' not in vdur:
- continue
- vdu = next(
- filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
- )
- if 'monitoring-param' in vdu:
- for param in vdu['monitoring-param']:
- metric_name = param['nfvi-metric']
- gnocchi_metric_name = METRIC_MAPPINGS[metric_name]
- delta = 10 * self.granularity
- start_date = datetime.datetime.now() - datetime.timedelta(seconds=delta)
- resource_id = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
- try:
- measures = self.gnocchi_client.metric.get_measures(gnocchi_metric_name,
- start=start_date,
- resource_id=resource_id,
- granularity=self.granularity)
- if len(measures):
- metric = Metric(nsr_id, vnf_member_index, vdur['name'], metric_name, measures[-1][2])
- metrics.append(metric)
- except gnocchiclient.exceptions.NotFound as e:
- log.debug("No metric found: %s", e)
- pass
- return metrics
+++ /dev/null
-# -*- coding: utf-8 -*-
-
-##
-# Copyright 2016-2017 VMware Inc.
-# This file is part of ETSI OSM
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact: osslegalrouting@vmware.com
-##
-
-import json
-import logging
-
-from pyvcloud.vcd.client import BasicLoginCredentials
-from pyvcloud.vcd.client import Client
-import requests
-import time
-from xml.etree import ElementTree as XmlElementTree
-import traceback
-import six
-
-from osm_mon.collector.collectors.base_vim import BaseVimCollector
-from osm_mon.collector.metric import Metric
-from osm_mon.core.auth import AuthManager
-from osm_mon.core.common_db import CommonDbClient
-from osm_mon.core.settings import Config
-
-log = logging.getLogger(__name__)
-
-API_VERSION = '5.9'
-
-PERIOD_MSEC = {'HR': 3600000,
- 'DAY': 86400000,
- 'WEEK': 604800000,
- 'MONTH': 2678400000,
- 'YEAR': 31536000000}
-
-METRIC_MAPPINGS = {
- "average_memory_utilization": "mem|usage_average",
- "cpu_utilization": "cpu|usage_average",
- "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average",
- "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average",
- "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average",
- "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average",
- "packets_dropped_0": "net:4000|dropped",
- "packets_dropped_1": "net:4001|dropped",
- "packets_dropped_2": "net:4002|dropped",
- "packets_received": "net:Aggregate of all instances|packetsRxPerSec",
- "packets_sent": "net:Aggregate of all instances|packetsTxPerSec",
-}
-
-# Disable warnings from self-signed certificates.
-requests.packages.urllib3.disable_warnings()
-
-
-class VMwareCollector(BaseVimCollector):
- def __init__(self, vim_account_id: str):
- super().__init__(vim_account_id)
- self.common_db = CommonDbClient()
- self.auth_manager = AuthManager()
- self.granularity = self._get_granularity(vim_account_id)
- vim_account = self.get_vim_account(vim_account_id)
- self.vrops_site = vim_account['vrops_site']
- self.vrops_user = vim_account['vrops_user']
- self.vrops_password = vim_account['vrops_password']
- self.vcloud_site = vim_account['vim_url']
- self.admin_username = vim_account['admin_username']
- self.admin_password = vim_account['admin_password']
- self.vim_uuid = vim_account['vim_uuid']
-
- def connect_as_admin(self):
- """ Method connect as pvdc admin user to vCloud director.
- There are certain action that can be done only by provider vdc admin user.
- Organization creation / provider network creation etc.
-
- Returns:
- The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
- """
-
- log.info("Logging into vCD org as admin.")
-
- try:
- host = self.vcloud_site
- admin_user = self.admin_username
- admin_passwd = self.admin_password
- org = 'System'
- client = Client(host, verify_ssl_certs=False)
- client.set_credentials(BasicLoginCredentials(admin_user, org,
- admin_passwd))
- return client
-
- except Exception as e:
- log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
-
- def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
- vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
- return vdur['vim-id']
-
- def get_vim_account(self, vim_account_id: str):
- """
- Method to get VIM account details by its ID
- arg - VIM ID
- return - dict with vim account details
- """
- vim_account = {}
- vim_account_info = self.auth_manager.get_credentials(vim_account_id)
-
- vim_account['name'] = vim_account_info.name
- vim_account['vim_tenant_name'] = vim_account_info.tenant_name
- vim_account['vim_type'] = vim_account_info.type
- vim_account['vim_url'] = vim_account_info.url
- vim_account['org_user'] = vim_account_info.user
- vim_account['org_password'] = vim_account_info.password
- vim_account['vim_uuid'] = vim_account_info.uuid
-
- vim_config = json.loads(vim_account_info.config)
- vim_account['admin_username'] = vim_config['admin_username']
- vim_account['admin_password'] = vim_config['admin_password']
- vim_account['vrops_site'] = vim_config['vrops_site']
- vim_account['vrops_user'] = vim_config['vrops_user']
- vim_account['vrops_password'] = vim_config['vrops_password']
- vim_account['vcenter_ip'] = vim_config['vcenter_ip']
- vim_account['vcenter_port'] = vim_config['vcenter_port']
- vim_account['vcenter_user'] = vim_config['vcenter_user']
- vim_account['vcenter_password'] = vim_config['vcenter_password']
-
- if vim_config['nsx_manager'] is not None:
- vim_account['nsx_manager'] = vim_config['nsx_manager']
-
- if vim_config['nsx_user'] is not None:
- vim_account['nsx_user'] = vim_config['nsx_user']
-
- if vim_config['nsx_password'] is not None:
- vim_account['nsx_password'] = vim_config['nsx_password']
-
- if vim_config['orgname'] is not None:
- vim_account['orgname'] = vim_config['orgname']
-
- return vim_account
-
- def _get_granularity(self, vim_account_id: str):
- creds = self.auth_manager.get_credentials(vim_account_id)
- vim_config = json.loads(creds.config)
- if 'granularity' in vim_config:
- return int(vim_config['granularity'])
- else:
- cfg = Config.instance()
- return cfg.OS_DEFAULT_GRANULARITY
-
- def get_vm_moref_id(self, vapp_uuid):
- """
- Method to get the moref_id of given VM
- arg - vapp_uuid
- return - VM mored_id
- """
- try:
- if vapp_uuid:
- vm_details = self.get_vapp_details_rest(vapp_uuid)
-
- if vm_details and "vm_vcenter_info" in vm_details:
- vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
-
- log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
- return vm_moref_id
-
- except Exception as exp:
- log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc()))
-
- def get_vapp_details_rest(self, vapp_uuid=None):
- """
- Method retrieve vapp detail from vCloud director
- vapp_uuid - is vapp identifier.
- Returns - VM MOref ID or return None
- """
- parsed_respond = {}
- vca = None
-
- if vapp_uuid is None:
- return parsed_respond
-
- vca = self.connect_as_admin()
-
- if not vca:
- log.error("Failed to connect to vCD")
- return parsed_respond
-
- url_list = [self.vcloud_site, '/api/vApp/vapp-', vapp_uuid]
- get_vapp_restcall = ''.join(url_list)
-
- if vca._session:
- headers = {'Accept': 'application/*+xml;version=' + API_VERSION,
- 'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
- response = requests.get(get_vapp_restcall,
- headers=headers,
- verify=False)
-
- if response.status_code != 200:
- log.error("REST API call {} failed. Return status code {}".format(get_vapp_restcall,
- response.content))
- return parsed_respond
-
- try:
- xmlroot_respond = XmlElementTree.fromstring(response.content)
-
- namespaces = {'vm': 'http://www.vmware.com/vcloud/v1.5',
- "vmext": "http://www.vmware.com/vcloud/extension/v1.5",
- "xmlns": "http://www.vmware.com/vcloud/v1.5"}
-
- # parse children section for other attrib
- children_section = xmlroot_respond.find('vm:Children/', namespaces)
- if children_section is not None:
- vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
- if vCloud_extension_section is not None:
- vm_vcenter_info = {}
- vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
- vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
- if vmext is not None:
- vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
- parsed_respond["vm_vcenter_info"] = vm_vcenter_info
-
- except Exception as exp:
- log.info("Error occurred for getting vApp details: {}\n{}".format(exp,
- traceback.format_exc())
- )
-
- return parsed_respond
-
- def get_vm_resource_id(self, vm_moref_id):
- """ Find resource ID in vROPs using vm_moref_id
- """
- if vm_moref_id is None:
- return None
-
- api_url = '/suite-api/api/resources?resourceKind=VirtualMachine'
- headers = {'Accept': 'application/json'}
-
- resp = requests.get(self.vrops_site + api_url,
- auth=(self.vrops_user, self.vrops_password),
- verify=False, headers=headers)
-
- if resp.status_code != 200:
- log.error("Failed to get resource details for{} {} {}".format(vm_moref_id,
- resp.status_code,
- resp.content))
- return None
-
- vm_resource_id = None
- try:
- resp_data = json.loads(resp.content.decode('utf-8'))
- if resp_data.get('resourceList') is not None:
- resource_list = resp_data.get('resourceList')
- for resource in resource_list:
- if resource.get('resourceKey') is not None:
- resource_details = resource['resourceKey']
- if resource_details.get('resourceIdentifiers') is not None:
- resource_identifiers = resource_details['resourceIdentifiers']
- for resource_identifier in resource_identifiers:
- if resource_identifier['identifierType']['name'] == 'VMEntityObjectID':
- if resource_identifier.get('value') is not None and \
- resource_identifier['value'] == vm_moref_id:
- vm_resource_id = resource['identifier']
- log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id,
- vm_moref_id))
-
- except Exception as exp:
- log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc()))
-
- return vm_resource_id
-
- def collect(self, vnfr: dict):
- nsr_id = vnfr['nsr-id-ref']
- vnf_member_index = vnfr['member-vnf-index-ref']
- vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
- metrics = []
- for vdur in vnfr['vdur']:
- # This avoids errors when vdur records have not been completely filled
- if 'name' not in vdur:
- continue
- vdu = next(
- filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
-
- )
- if 'monitoring-param' in vdu:
- for param in vdu['monitoring-param']:
- metric_name = param['nfvi-metric']
- vrops_metric_name = METRIC_MAPPINGS[metric_name]
- resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
-
- # Find vm_moref_id from vApp uuid in vCD
- vm_moref_id = self.get_vm_moref_id(resource_uuid)
- if vm_moref_id is None:
- log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid))
- return
-
- # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
- resource_id = self.get_vm_resource_id(vm_moref_id)
- if resource_id is None:
- log.debug("Failed to find resource in vROPs: {}".format(resource_uuid))
- return
- try:
- end_time = int(round(time.time() * 1000))
- time_diff = PERIOD_MSEC['YEAR']
- begin_time = end_time - time_diff
-
- api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format(
- resource_id, vrops_metric_name, str(begin_time), str(end_time))
-
- headers = {'Accept': 'application/json'}
-
- resp = requests.get(self.vrops_site + api_url,
- auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
- )
-
- if resp.status_code != 200:
- log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name,
- resp.status_code,
- resp.content))
- return
-
- metrics_data = {}
- m_data = json.loads(resp.content.decode('utf-8'))
-
- for resp_key, resp_val in six.iteritems(m_data):
- if resp_key == 'values':
- data = m_data['values'][0]
- for data_k, data_v in six.iteritems(data):
- if data_k == 'stat-list':
- stat_list = data_v
- for stat_list_k, stat_list_v in six.iteritems(stat_list):
- for stat_keys, stat_vals in six.iteritems(stat_list_v[0]):
- if stat_keys == 'timestamps':
- metrics_data['time_series'] = stat_list_v[0]['timestamps']
- if stat_keys == 'data':
- metrics_data['metrics_series'] = stat_list_v[0]['data']
-
- if metrics_data:
- metric = Metric(nsr_id,
- vnf_member_index,
- vdur['name'],
- metric_name,
- metrics_data['metrics_series'][-1])
-
- metrics.append(metric)
-
- except Exception as e:
- log.debug("No metric found: %s", e)
- pass
- return metrics
--- /dev/null
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
--- /dev/null
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+
+
+class BaseInfraCollector:
+ pass
--- /dev/null
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+from osm_mon.collector.infra_collectors.base import BaseInfraCollector
+
+
+class BaseVimInfraCollector(BaseInfraCollector):
+ def __init__(self, vim_account_id: str):
+ pass
+
+ def is_vim_ok(self) -> bool:
+ pass
--- /dev/null
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+import logging
+
+from keystoneauth1 import session
+from keystoneauth1.identity import v3
+from keystoneclient.v3 import client
+
+from osm_mon.collector.infra_collectors.base_vim import BaseVimInfraCollector
+from osm_mon.core.auth import AuthManager
+
+log = logging.getLogger(__name__)
+
+
+class OpenstackInfraCollector(BaseVimInfraCollector):
+ def __init__(self, vim_account_id: str):
+ super().__init__(vim_account_id)
+ self.auth_manager = AuthManager()
+ self.keystone_client = self._build_keystone_client(vim_account_id)
+
+ def is_vim_ok(self) -> bool:
+ try:
+ self.keystone_client.projects.list()
+ return True
+ except Exception:
+ log.exception("VIM status is not OK!")
+ return False
+
+ def _build_keystone_client(self, vim_account_id):
+ creds = self.auth_manager.get_credentials(vim_account_id)
+ verify_ssl = self.auth_manager.is_verify_ssl(vim_account_id)
+ auth = v3.Password(auth_url=creds.url,
+ username=creds.user,
+ password=creds.password,
+ project_name=creds.tenant_name,
+ project_domain_id='default',
+ user_domain_id='default')
+ sess = session.Session(auth=auth, verify=verify_ssl)
+ return client.Client(session=sess)
class Metric:
- def __init__(self, nsr_id, vnf_member_index, vdur_name, name, value):
- self.nsr_id = nsr_id
- self.vnf_member_index = vnf_member_index
- self.vdur_name = vdur_name
+ def __init__(self, tags: dict, name: str, value):
+ self.tags = tags
self.name = name
self.value = value
--- /dev/null
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
--- /dev/null
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+from typing import List
+
+from osm_mon.collector.metric import Metric
+
+
+class BaseCollector:
+ def collect(self, vnfr: dict) -> List[Metric]:
+ pass
--- /dev/null
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+
+from osm_mon.collector.vnf_collectors.base import BaseCollector
+
+
+class BaseVimCollector(BaseCollector):
+ def __init__(self, vim_account_id: str):
+ pass
--- /dev/null
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+import asyncio
+import logging
+from typing import List
+
+from n2vc.vnf import N2VC
+
+from osm_mon.collector.metric import Metric
+from osm_mon.collector.vnf_collectors.base import BaseCollector
+from osm_mon.collector.vnf_metric import VnfMetric
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.exceptions import VcaDeploymentInfoNotFound
+from osm_mon.core.settings import Config
+
+log = logging.getLogger(__name__)
+
+
+class VCACollector(BaseCollector):
+ def __init__(self):
+ cfg = Config.instance()
+ self.common_db = CommonDbClient()
+ self.loop = asyncio.get_event_loop()
+ self.n2vc = N2VC(server=cfg.OSMMON_VCA_HOST, user=cfg.OSMMON_VCA_USER, secret=cfg.OSMMON_VCA_SECRET)
+
+ def collect(self, vnfr: dict) -> List[Metric]:
+ nsr_id = vnfr['nsr-id-ref']
+ vnf_member_index = vnfr['member-vnf-index-ref']
+ vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
+ metrics = []
+ for vdur in vnfr['vdur']:
+ # This avoids errors when vdur records have not been completely filled
+ if 'name' not in vdur:
+ continue
+ vdu = next(
+ filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
+ )
+ if 'vdu-configuration' in vdu and 'metrics' in vdu['vdu-configuration']:
+ try:
+ vca_deployment_info = self.get_vca_deployment_info(nsr_id, vnf_member_index, vdur['name'])
+ except VcaDeploymentInfoNotFound:
+ continue
+ measures = self.loop.run_until_complete(self.n2vc.GetMetrics(vca_deployment_info['model'],
+ vca_deployment_info['application']))
+ log.debug('Measures: %s', measures)
+ for measure_list in measures.values():
+ for measure in measure_list:
+ log.debug("Measure: %s", measure)
+ metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], measure['key'],
+ float(measure['value']))
+ metrics.append(metric)
+ if 'vnf-configuration' in vnfd and 'metrics' in vnfd['vnf-configuration']:
+ try:
+ vca_deployment_info = self.get_vca_deployment_info(nsr_id, vnf_member_index, None)
+ except VcaDeploymentInfoNotFound:
+ return metrics
+ measures = self.loop.run_until_complete(self.n2vc.GetMetrics(vca_deployment_info['model'],
+ vca_deployment_info['application']))
+ log.debug('Measures: %s', measures)
+ for measure_list in measures.values():
+ for measure in measure_list:
+ log.debug("Measure: %s", measure)
+ metric = VnfMetric(nsr_id, vnf_member_index, '', measure['key'], float(measure['value']))
+ metrics.append(metric)
+ return metrics
+
+ def get_vca_deployment_info(self, nsr_id, vnf_member_index, vdur_name):
+ nsr = self.common_db.get_nsr(nsr_id)
+ for vca_deployment in nsr["_admin"]["deployed"]["VCA"]:
+ if vca_deployment:
+ if vca_deployment['member-vnf-index'] == vnf_member_index and vca_deployment['vdu_name'] == vdur_name:
+ return vca_deployment
+ raise VcaDeploymentInfoNotFound("VCA deployment info for nsr_id {}, index {} and vdur_name {} not found."
+ .format(nsr_id, vnf_member_index, vdur_name))
--- /dev/null
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+import datetime
+import json
+import logging
+from typing import List
+
+import gnocchiclient.exceptions
+from gnocchiclient.v1 import client as gnocchi_client
+from keystoneauth1 import session
+from keystoneauth1.identity import v3
+
+from osm_mon.collector.metric import Metric
+from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
+from osm_mon.collector.vnf_metric import VnfMetric
+from osm_mon.core.auth import AuthManager
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.settings import Config
+
+log = logging.getLogger(__name__)
+
+METRIC_MAPPINGS = {
+ "average_memory_utilization": "memory.usage",
+ "disk_read_ops": "disk.read.requests",
+ "disk_write_ops": "disk.write.requests",
+ "disk_read_bytes": "disk.read.bytes",
+ "disk_write_bytes": "disk.write.bytes",
+ "packets_dropped": "interface.if_dropped",
+ "packets_received": "interface.if_packets",
+ "packets_sent": "interface.if_packets",
+ "cpu_utilization": "cpu_util",
+}
+
+
+class OpenstackCollector(BaseVimCollector):
+ def __init__(self, vim_account_id: str):
+ super().__init__(vim_account_id)
+ self.common_db = CommonDbClient()
+ self.auth_manager = AuthManager()
+ self.granularity = self._get_granularity(vim_account_id)
+ self.gnocchi_client = self._build_gnocchi_client(vim_account_id)
+
+ def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
+ vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
+ return vdur['vim-id']
+
+ def _build_gnocchi_client(self, vim_account_id: str) -> gnocchi_client.Client:
+ creds = self.auth_manager.get_credentials(vim_account_id)
+ verify_ssl = self.auth_manager.is_verify_ssl(vim_account_id)
+ auth = v3.Password(auth_url=creds.url,
+ username=creds.user,
+ password=creds.password,
+ project_name=creds.tenant_name,
+ project_domain_id='default',
+ user_domain_id='default')
+ sess = session.Session(auth=auth, verify=verify_ssl)
+ return gnocchi_client.Client(session=sess)
+
+ def _get_granularity(self, vim_account_id: str):
+ creds = self.auth_manager.get_credentials(vim_account_id)
+ vim_config = json.loads(creds.config)
+ if 'granularity' in vim_config:
+ return int(vim_config['granularity'])
+ else:
+ cfg = Config.instance()
+ return cfg.OS_DEFAULT_GRANULARITY
+
+ def collect(self, vnfr: dict) -> List[Metric]:
+ nsr_id = vnfr['nsr-id-ref']
+ vnf_member_index = vnfr['member-vnf-index-ref']
+ vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
+ metrics = []
+ for vdur in vnfr['vdur']:
+ # This avoids errors when vdur records have not been completely filled
+ if 'name' not in vdur:
+ continue
+ vdu = next(
+ filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
+ )
+ if 'monitoring-param' in vdu:
+ for param in vdu['monitoring-param']:
+ metric_name = param['nfvi-metric']
+ gnocchi_metric_name = METRIC_MAPPINGS[metric_name]
+ delta = 10 * self.granularity
+ start_date = datetime.datetime.now() - datetime.timedelta(seconds=delta)
+ resource_id = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
+ try:
+ measures = self.gnocchi_client.metric.get_measures(gnocchi_metric_name,
+ start=start_date,
+ resource_id=resource_id,
+ granularity=self.granularity)
+ if len(measures):
+ metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], metric_name, measures[-1][2])
+ metrics.append(metric)
+ except gnocchiclient.exceptions.NotFound as e:
+ log.debug("No metric found: %s", e)
+ pass
+ return metrics
--- /dev/null
+# -*- coding: utf-8 -*-
+
+##
+# Copyright 2016-2017 VMware Inc.
+# This file is part of ETSI OSM
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: osslegalrouting@vmware.com
+##
+
+import json
+import logging
+import time
+import traceback
+from xml.etree import ElementTree as XmlElementTree
+
+import requests
+import six
+from pyvcloud.vcd.client import BasicLoginCredentials
+from pyvcloud.vcd.client import Client
+
+from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
+from osm_mon.collector.vnf_metric import VnfMetric
+from osm_mon.core.auth import AuthManager
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.settings import Config
+
+log = logging.getLogger(__name__)
+
+API_VERSION = '5.9'
+
+PERIOD_MSEC = {'HR': 3600000,
+ 'DAY': 86400000,
+ 'WEEK': 604800000,
+ 'MONTH': 2678400000,
+ 'YEAR': 31536000000}
+
+METRIC_MAPPINGS = {
+ "average_memory_utilization": "mem|usage_average",
+ "cpu_utilization": "cpu|usage_average",
+ "read_latency_0": "virtualDisk:scsi0:0|totalReadLatency_average",
+ "write_latency_0": "virtualDisk:scsi0:0|totalWriteLatency_average",
+ "read_latency_1": "virtualDisk:scsi0:1|totalReadLatency_average",
+ "write_latency_1": "virtualDisk:scsi0:1|totalWriteLatency_average",
+ "packets_dropped_0": "net:4000|dropped",
+ "packets_dropped_1": "net:4001|dropped",
+ "packets_dropped_2": "net:4002|dropped",
+ "packets_received": "net:Aggregate of all instances|packetsRxPerSec",
+ "packets_sent": "net:Aggregate of all instances|packetsTxPerSec",
+}
+
+# Disable warnings from self-signed certificates.
+requests.packages.urllib3.disable_warnings()
+
+
+class VMwareCollector(BaseVimCollector):
+ def __init__(self, vim_account_id: str):
+ super().__init__(vim_account_id)
+ self.common_db = CommonDbClient()
+ self.auth_manager = AuthManager()
+ self.granularity = self._get_granularity(vim_account_id)
+ vim_account = self.get_vim_account(vim_account_id)
+ self.vrops_site = vim_account['vrops_site']
+ self.vrops_user = vim_account['vrops_user']
+ self.vrops_password = vim_account['vrops_password']
+ self.vcloud_site = vim_account['vim_url']
+ self.admin_username = vim_account['admin_username']
+ self.admin_password = vim_account['admin_password']
+ self.vim_uuid = vim_account['vim_uuid']
+
+ def connect_as_admin(self):
+ """ Method connect as pvdc admin user to vCloud director.
+ There are certain action that can be done only by provider vdc admin user.
+ Organization creation / provider network creation etc.
+
+ Returns:
+ The return client object that letter can be used to connect to vcloud direct as admin for provider vdc
+ """
+
+ log.info("Logging into vCD org as admin.")
+
+ try:
+ host = self.vcloud_site
+ admin_user = self.admin_username
+ admin_passwd = self.admin_password
+ org = 'System'
+ client = Client(host, verify_ssl_certs=False)
+ client.set_credentials(BasicLoginCredentials(admin_user, org,
+ admin_passwd))
+ return client
+
+ except Exception as e:
+ log.info("Can't connect to a vCloud director as: {} with exception {}".format(admin_user, e))
+
+ def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str:
+ vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
+ return vdur['vim-id']
+
+ def get_vim_account(self, vim_account_id: str):
+ """
+ Method to get VIM account details by its ID
+ arg - VIM ID
+ return - dict with vim account details
+ """
+ vim_account = {}
+ vim_account_info = self.auth_manager.get_credentials(vim_account_id)
+
+ vim_account['name'] = vim_account_info.name
+ vim_account['vim_tenant_name'] = vim_account_info.tenant_name
+ vim_account['vim_type'] = vim_account_info.type
+ vim_account['vim_url'] = vim_account_info.url
+ vim_account['org_user'] = vim_account_info.user
+ vim_account['org_password'] = vim_account_info.password
+ vim_account['vim_uuid'] = vim_account_info.uuid
+
+ vim_config = json.loads(vim_account_info.config)
+ vim_account['admin_username'] = vim_config['admin_username']
+ vim_account['admin_password'] = vim_config['admin_password']
+ vim_account['vrops_site'] = vim_config['vrops_site']
+ vim_account['vrops_user'] = vim_config['vrops_user']
+ vim_account['vrops_password'] = vim_config['vrops_password']
+ vim_account['vcenter_ip'] = vim_config['vcenter_ip']
+ vim_account['vcenter_port'] = vim_config['vcenter_port']
+ vim_account['vcenter_user'] = vim_config['vcenter_user']
+ vim_account['vcenter_password'] = vim_config['vcenter_password']
+
+ if vim_config['nsx_manager'] is not None:
+ vim_account['nsx_manager'] = vim_config['nsx_manager']
+
+ if vim_config['nsx_user'] is not None:
+ vim_account['nsx_user'] = vim_config['nsx_user']
+
+ if vim_config['nsx_password'] is not None:
+ vim_account['nsx_password'] = vim_config['nsx_password']
+
+ if vim_config['orgname'] is not None:
+ vim_account['orgname'] = vim_config['orgname']
+
+ return vim_account
+
+ def _get_granularity(self, vim_account_id: str):
+ creds = self.auth_manager.get_credentials(vim_account_id)
+ vim_config = json.loads(creds.config)
+ if 'granularity' in vim_config:
+ return int(vim_config['granularity'])
+ else:
+ cfg = Config.instance()
+ return cfg.OS_DEFAULT_GRANULARITY
+
+ def get_vm_moref_id(self, vapp_uuid):
+ """
+ Method to get the moref_id of given VM
+ arg - vapp_uuid
+ return - VM mored_id
+ """
+ try:
+ if vapp_uuid:
+ vm_details = self.get_vapp_details_rest(vapp_uuid)
+
+ if vm_details and "vm_vcenter_info" in vm_details:
+ vm_moref_id = vm_details["vm_vcenter_info"].get("vm_moref_id", None)
+
+ log.info("Found vm_moref_id: {} for vApp UUID: {}".format(vm_moref_id, vapp_uuid))
+ return vm_moref_id
+
+ except Exception as exp:
+ log.info("Error occurred while getting VM moref ID for VM : {}\n{}".format(exp, traceback.format_exc()))
+
+ def get_vapp_details_rest(self, vapp_uuid=None):
+ """
+ Method retrieve vapp detail from vCloud director
+ vapp_uuid - is vapp identifier.
+ Returns - VM MOref ID or return None
+ """
+ parsed_respond = {}
+ vca = None
+
+ if vapp_uuid is None:
+ return parsed_respond
+
+ vca = self.connect_as_admin()
+
+ if not vca:
+ log.error("Failed to connect to vCD")
+ return parsed_respond
+
+ url_list = [self.vcloud_site, '/api/vApp/vapp-', vapp_uuid]
+ get_vapp_restcall = ''.join(url_list)
+
+ if vca._session:
+ headers = {'Accept': 'application/*+xml;version=' + API_VERSION,
+ 'x-vcloud-authorization': vca._session.headers['x-vcloud-authorization']}
+ response = requests.get(get_vapp_restcall,
+ headers=headers,
+ verify=False)
+
+ if response.status_code != 200:
+ log.error("REST API call {} failed. Return status code {}".format(get_vapp_restcall,
+ response.content))
+ return parsed_respond
+
+ try:
+ xmlroot_respond = XmlElementTree.fromstring(response.content)
+
+ namespaces = {'vm': 'http://www.vmware.com/vcloud/v1.5',
+ "vmext": "http://www.vmware.com/vcloud/extension/v1.5",
+ "xmlns": "http://www.vmware.com/vcloud/v1.5"}
+
+ # parse children section for other attrib
+ children_section = xmlroot_respond.find('vm:Children/', namespaces)
+ if children_section is not None:
+ vCloud_extension_section = children_section.find('xmlns:VCloudExtension', namespaces)
+ if vCloud_extension_section is not None:
+ vm_vcenter_info = {}
+ vim_info = vCloud_extension_section.find('vmext:VmVimInfo', namespaces)
+ vmext = vim_info.find('vmext:VmVimObjectRef', namespaces)
+ if vmext is not None:
+ vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text
+ parsed_respond["vm_vcenter_info"] = vm_vcenter_info
+
+ except Exception as exp:
+ log.info("Error occurred for getting vApp details: {}\n{}".format(exp,
+ traceback.format_exc())
+ )
+
+ return parsed_respond
+
+ def get_vm_resource_id(self, vm_moref_id):
+ """ Find resource ID in vROPs using vm_moref_id
+ """
+ if vm_moref_id is None:
+ return None
+
+ api_url = '/suite-api/api/resources?resourceKind=VirtualMachine'
+ headers = {'Accept': 'application/json'}
+
+ resp = requests.get(self.vrops_site + api_url,
+ auth=(self.vrops_user, self.vrops_password),
+ verify=False, headers=headers)
+
+ if resp.status_code != 200:
+ log.error("Failed to get resource details for{} {} {}".format(vm_moref_id,
+ resp.status_code,
+ resp.content))
+ return None
+
+ vm_resource_id = None
+ try:
+ resp_data = json.loads(resp.content.decode('utf-8'))
+ if resp_data.get('resourceList') is not None:
+ resource_list = resp_data.get('resourceList')
+ for resource in resource_list:
+ if resource.get('resourceKey') is not None:
+ resource_details = resource['resourceKey']
+ if resource_details.get('resourceIdentifiers') is not None:
+ resource_identifiers = resource_details['resourceIdentifiers']
+ for resource_identifier in resource_identifiers:
+ if resource_identifier['identifierType']['name'] == 'VMEntityObjectID':
+ if resource_identifier.get('value') is not None and \
+ resource_identifier['value'] == vm_moref_id:
+ vm_resource_id = resource['identifier']
+ log.info("Found VM resource ID: {} for vm_moref_id: {}".format(vm_resource_id,
+ vm_moref_id))
+
+ except Exception as exp:
+ log.info("get_vm_resource_id: Error in parsing {}\n{}".format(exp, traceback.format_exc()))
+
+ return vm_resource_id
+
+ def collect(self, vnfr: dict):
+ nsr_id = vnfr['nsr-id-ref']
+ vnf_member_index = vnfr['member-vnf-index-ref']
+ vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
+ metrics = []
+ for vdur in vnfr['vdur']:
+ # This avoids errors when vdur records have not been completely filled
+ if 'name' not in vdur:
+ continue
+ vdu = next(
+ filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
+
+ )
+ if 'monitoring-param' in vdu:
+ for param in vdu['monitoring-param']:
+ metric_name = param['nfvi-metric']
+ vrops_metric_name = METRIC_MAPPINGS[metric_name]
+ resource_uuid = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
+
+ # Find vm_moref_id from vApp uuid in vCD
+ vm_moref_id = self.get_vm_moref_id(resource_uuid)
+ if vm_moref_id is None:
+ log.debug("Failed to find vm morefid for vApp in vCD: {}".format(resource_uuid))
+ return
+
+ # Based on vm_moref_id, find VM's corresponding resource_id in vROPs
+ resource_id = self.get_vm_resource_id(vm_moref_id)
+ if resource_id is None:
+ log.debug("Failed to find resource in vROPs: {}".format(resource_uuid))
+ return
+ try:
+ end_time = int(round(time.time() * 1000))
+ time_diff = PERIOD_MSEC['YEAR']
+ begin_time = end_time - time_diff
+
+ api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format(
+ resource_id, vrops_metric_name, str(begin_time), str(end_time))
+
+ headers = {'Accept': 'application/json'}
+
+ resp = requests.get(self.vrops_site + api_url,
+ auth=(self.vrops_user, self.vrops_password), verify=False, headers=headers
+ )
+
+ if resp.status_code != 200:
+ log.info("Failed to get Metrics data from vROPS for {} {} {}".format(vrops_metric_name,
+ resp.status_code,
+ resp.content))
+ return
+
+ metrics_data = {}
+ m_data = json.loads(resp.content.decode('utf-8'))
+
+ for resp_key, resp_val in six.iteritems(m_data):
+ if resp_key == 'values':
+ data = m_data['values'][0]
+ for data_k, data_v in six.iteritems(data):
+ if data_k == 'stat-list':
+ stat_list = data_v
+ for stat_list_k, stat_list_v in six.iteritems(stat_list):
+ for stat_keys, stat_vals in six.iteritems(stat_list_v[0]):
+ if stat_keys == 'timestamps':
+ metrics_data['time_series'] = stat_list_v[0]['timestamps']
+ if stat_keys == 'data':
+ metrics_data['metrics_series'] = stat_list_v[0]['data']
+
+ if metrics_data:
+ metric = VnfMetric(nsr_id,
+ vnf_member_index,
+ vdur['name'],
+ metric_name,
+ metrics_data['metrics_series'][-1])
+
+ metrics.append(metric)
+
+ except Exception as e:
+ log.debug("No metric found: %s", e)
+ pass
+ return metrics
--- /dev/null
+# Copyright 2018 Whitestack, LLC
+# *************************************************************
+
+# This file is part of OSM Monitoring module
+# All Rights Reserved to Whitestack, LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact: bdiaz@whitestack.com or glavado@whitestack.com
+##
+from osm_mon.collector.metric import Metric
+
+
+class VnfMetric(Metric):
+ def __init__(self, nsr_id, vnf_member_index, vdur_name, name, value):
+ tags = {
+ 'ns_id': nsr_id,
+ 'vnf_member_index': vnf_member_index,
+ 'vdu_name': vdur_name
+ }
+ super().__init__(tags, name, value)
def get_vim_account_id(self, nsr_id: str, vnf_member_index: int) -> str:
vnfr = self.get_vnfr(nsr_id, vnf_member_index)
return vnfr['vim-account-id']
+
+ def get_vim_accounts(self):
+ return self.common_db.get_list('vim_accounts')
import logging
import uuid
-from peewee import CharField, TextField, FloatField, Model, AutoField
+from peewee import CharField, TextField, FloatField, Model, AutoField, Proxy
from playhouse.db_url import connect
from osm_mon.core.settings import Config
log = logging.getLogger(__name__)
-cfg = Config.instance()
-cfg.read_environ()
-db = connect(cfg.DATABASE)
+db = Proxy()
class BaseModel(Model):
class DatabaseManager:
+ def __init__(self):
+ cfg = Config.instance()
+ cfg.read_environ()
+ db.initialize(connect(cfg.DATABASE))
+
def create_tables(self) -> None:
- try:
- db.connect()
+ with db.atomic():
db.create_tables([VimCredentials, Alarm])
- db.close()
- except Exception:
- log.exception("Error creating tables: ")
- def get_credentials(self, vim_uuid) -> VimCredentials:
- return VimCredentials.get_or_none(VimCredentials.uuid == vim_uuid)
+ def get_credentials(self, vim_uuid: str = None) -> VimCredentials:
+ with db.atomic():
+ return VimCredentials.get_or_none(VimCredentials.uuid == vim_uuid)
def save_credentials(self, vim_credentials) -> VimCredentials:
"""Saves vim credentials. If a record with same uuid exists, overwrite it."""
- exists = VimCredentials.get_or_none(VimCredentials.uuid == vim_credentials.uuid)
- if exists:
- vim_credentials.id = exists.id
- vim_credentials.save()
- return vim_credentials
+ with db.atomic():
+ exists = VimCredentials.get_or_none(VimCredentials.uuid == vim_credentials.uuid)
+ if exists:
+ vim_credentials.id = exists.id
+ vim_credentials.save()
+ return vim_credentials
def get_alarm(self, alarm_id) -> Alarm:
- alarm = (Alarm.select()
- .where(Alarm.alarm_id == alarm_id)
- .get())
- return alarm
+ with db.atomic():
+ alarm = (Alarm.select()
+ .where(Alarm.alarm_id == alarm_id)
+ .get())
+ return alarm
def save_alarm(self, name, threshold, operation, severity, statistic, metric_name, vdur_name,
vnf_member_index, nsr_id) -> Alarm:
"""Saves alarm."""
# TODO: Add uuid optional param and check if exists to handle updates (see self.save_credentials)
- alarm = Alarm()
- alarm.uuid = str(uuid.uuid4())
- alarm.name = name
- alarm.threshold = threshold
- alarm.operation = operation
- alarm.severity = severity
- alarm.statistic = statistic
- alarm.monitoring_param = metric_name
- alarm.vdur_name = vdur_name
- alarm.vnf_member_index = vnf_member_index
- alarm.nsr_id = nsr_id
- alarm.save()
- return alarm
+ with db.atomic():
+ alarm = Alarm()
+ alarm.uuid = str(uuid.uuid4())
+ alarm.name = name
+ alarm.threshold = threshold
+ alarm.operation = operation
+ alarm.severity = severity
+ alarm.statistic = statistic
+ alarm.monitoring_param = metric_name
+ alarm.vdur_name = vdur_name
+ alarm.vnf_member_index = vnf_member_index
+ alarm.nsr_id = nsr_id
+ alarm.save()
+ return alarm
def delete_alarm(self, alarm_uuid) -> None:
- alarm = (Alarm.select()
- .where(Alarm.uuid == alarm_uuid)
- .get())
- alarm.delete_instance()
+ with db.atomic():
+ alarm = (Alarm.select()
+ .where(Alarm.uuid == alarm_uuid)
+ .get())
+ alarm.delete_instance()
def get_vim_type(self, vim_account_id) -> str:
"""Get the vim type that is required by the message."""
from unittest import mock
from osm_mon.collector.collector import Collector
-from osm_mon.collector.collectors.openstack import OpenstackCollector
+from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector
from osm_mon.core.database import DatabaseManager, db
projects / osm / MON.git / commitdiff