From: Benjamin Diaz Date: Wed, 10 Apr 2019 18:22:22 +0000 (-0300) Subject: Refactors code and adds unit tests X-Git-Tag: v6.0.0~13 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=a97bdb3eafa4f3d07d61d32635f7f36f5cc36c58;p=osm%2FMON.git Refactors code and adds unit tests Change-Id: Ia14034679af8fc7e9ac91a1bfb766e4b99ec9e39 Signed-off-by: Benjamin Diaz --- diff --git a/MANIFEST.in b/MANIFEST.in index 6e7058b..845273b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -20,7 +20,7 @@ # contact: prithiv.mohan@intel.com or adrian.hoban@intel.com. include requirements.txt +include test-requirements.txt include README.rst recursive-include osm_mon *.py *.xml *.sh *.yaml recursive-include devops-stages * -recursive-include test *.py diff --git a/osm_mon/collector/collector.py b/osm_mon/collector/collector.py index f4ffba4..cc84436 100644 --- a/osm_mon/collector/collector.py +++ b/osm_mon/collector/collector.py @@ -21,35 +21,16 @@ # contact: bdiaz@whitestack.com or glavado@whitestack.com ## import logging -import multiprocessing import time import peewee from osm_mon.collector.backends.prometheus import PrometheusBackend -from osm_mon.collector.infra_collectors.onos import OnosInfraCollector -from osm_mon.collector.infra_collectors.openstack import OpenstackInfraCollector -from osm_mon.collector.vnf_collectors.juju import VCACollector -from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector -from osm_mon.collector.vnf_collectors.vmware import VMwareCollector -from osm_mon.collector.vnf_collectors.vio import VIOCollector -from osm_mon.core.common_db import CommonDbClient +from osm_mon.collector.service import CollectorService from osm_mon.core.config import Config -from osm_mon.core.database import DatabaseManager log = logging.getLogger(__name__) -VIM_COLLECTORS = { - "openstack": OpenstackCollector, - "vmware": VMwareCollector, - "vio": VIOCollector -} -VIM_INFRA_COLLECTORS = { - "openstack": OpenstackInfraCollector -} -SDN_INFRA_COLLECTORS = { - "onos": OnosInfraCollector -} METRIC_BACKENDS = [ PrometheusBackend ] @@ -58,11 +39,8 @@ METRIC_BACKENDS = [ class Collector: def __init__(self, config: Config): self.conf = config - self.common_db = CommonDbClient(self.conf) - self.plugins = [] - self.database_manager = DatabaseManager(self.conf) - self.database_manager.create_tables() - self.queue = multiprocessing.Queue() + self.service = CollectorService(config) + self.backends = [] self._init_backends() def collect_forever(self): @@ -77,83 +55,11 @@ class Collector: except Exception: log.exception("Error collecting metrics") - def _collect_vim_metrics(self, vnfr: dict, vim_account_id: str): - # TODO(diazb) Add support for aws - database_manager = DatabaseManager(self.conf) - vim_type = database_manager.get_vim_type(vim_account_id) - if vim_type in VIM_COLLECTORS: - collector = VIM_COLLECTORS[vim_type](self.conf, vim_account_id) - metrics = collector.collect(vnfr) - for metric in metrics: - self.queue.put(metric) - else: - log.debug("vimtype %s is not supported.", vim_type) - - def _collect_vim_infra_metrics(self, vim_account_id: str): - database_manager = DatabaseManager(self.conf) - vim_type = database_manager.get_vim_type(vim_account_id) - if vim_type in VIM_INFRA_COLLECTORS: - collector = VIM_INFRA_COLLECTORS[vim_type](self.conf, vim_account_id) - metrics = collector.collect() - for metric in metrics: - self.queue.put(metric) - else: - log.debug("vimtype %s is not supported.", vim_type) - - def _collect_sdnc_infra_metrics(self, sdnc_id: str): - common_db = CommonDbClient(self.conf) - sdn_type = common_db.get_sdnc(sdnc_id)['type'] - if sdn_type in SDN_INFRA_COLLECTORS: - collector = SDN_INFRA_COLLECTORS[sdn_type](self.conf, sdnc_id) - metrics = collector.collect() - for metric in metrics: - self.queue.put(metric) - else: - log.debug("sdn_type %s is not supported.", sdn_type) - - def _collect_vca_metrics(self, vnfr: dict): - log.debug('_collect_vca_metrics') - log.debug('vnfr: %s', vnfr) - vca_collector = VCACollector(self.conf) - metrics = vca_collector.collect(vnfr) - for metric in metrics: - self.queue.put(metric) - def collect_metrics(self): - vnfrs = self.common_db.get_vnfrs() - processes = [] - for vnfr in vnfrs: - nsr_id = vnfr['nsr-id-ref'] - vnf_member_index = vnfr['member-vnf-index-ref'] - vim_account_id = self.common_db.get_vim_account_id(nsr_id, vnf_member_index) - p = multiprocessing.Process(target=self._collect_vim_metrics, - args=(vnfr, vim_account_id)) - processes.append(p) - p.start() - p = multiprocessing.Process(target=self._collect_vca_metrics, - args=(vnfr,)) - processes.append(p) - p.start() - vims = self.common_db.get_vim_accounts() - for vim in vims: - p = multiprocessing.Process(target=self._collect_vim_infra_metrics, - args=(vim['_id'],)) - processes.append(p) - p.start() - sdncs = self.common_db.get_sdncs() - for sdnc in sdncs: - p = multiprocessing.Process(target=self._collect_sdnc_infra_metrics, - args=(sdnc['_id'],)) - processes.append(p) - p.start() - for process in processes: - process.join(timeout=10) - metrics = [] - while not self.queue.empty(): - metrics.append(self.queue.get()) - for plugin in self.plugins: - plugin.handle(metrics) + metrics = self.service.collect_metrics() + for backend in self.backends: + backend.handle(metrics) def _init_backends(self): for backend in METRIC_BACKENDS: - self.plugins.append(backend()) + self.backends.append(backend()) diff --git a/osm_mon/collector/infra_collectors/openstack.py b/osm_mon/collector/infra_collectors/openstack.py index cb8259e..6086dd3 100644 --- a/osm_mon/collector/infra_collectors/openstack.py +++ b/osm_mon/collector/infra_collectors/openstack.py @@ -30,7 +30,7 @@ from novaclient import v2 as nova_client_v2 from osm_mon.collector.infra_collectors.base_vim import BaseVimInfraCollector from osm_mon.collector.metric import Metric -from osm_mon.core.auth import AuthManager +from osm_mon.collector.utils import CollectorUtils from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config @@ -40,7 +40,6 @@ log = logging.getLogger(__name__) class OpenstackInfraCollector(BaseVimInfraCollector): def __init__(self, config: Config, vim_account_id: str): super().__init__(config, vim_account_id) - self.auth_manager = AuthManager(config) self.keystone = self._build_keystone_client(vim_account_id) self.nova = self._build_nova_client(vim_account_id) self.vim_account_id = vim_account_id @@ -91,9 +90,9 @@ class OpenstackInfraCollector(BaseVimInfraCollector): sess = self._get_session(vim_account_id) return nova_client.Client("2", session=sess) - def _get_session(self, vim_account_id): - creds = self.auth_manager.get_credentials(vim_account_id) - verify_ssl = self.auth_manager.is_verify_ssl(vim_account_id) + def _get_session(self, vim_account_id: str): + creds = CollectorUtils.get_credentials(vim_account_id) + verify_ssl = CollectorUtils.is_verify_ssl(creds) auth = v3.Password(auth_url=creds.url, username=creds.user, password=creds.password, diff --git a/osm_mon/collector/service.py b/osm_mon/collector/service.py new file mode 100644 index 0000000..0724822 --- /dev/null +++ b/osm_mon/collector/service.py @@ -0,0 +1,109 @@ +import logging +import multiprocessing +from typing import List + +from osm_mon.collector.infra_collectors.onos import OnosInfraCollector +from osm_mon.collector.infra_collectors.openstack import OpenstackInfraCollector +from osm_mon.collector.metric import Metric +from osm_mon.collector.utils import CollectorUtils +from osm_mon.collector.vnf_collectors.juju import VCACollector +from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector +from osm_mon.collector.vnf_collectors.vio import VIOCollector +from osm_mon.collector.vnf_collectors.vmware import VMwareCollector +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config + +log = logging.getLogger(__name__) + +VIM_COLLECTORS = { + "openstack": OpenstackCollector, + "vmware": VMwareCollector, + "vio": VIOCollector +} +VIM_INFRA_COLLECTORS = { + "openstack": OpenstackInfraCollector +} +SDN_INFRA_COLLECTORS = { + "onos": OnosInfraCollector +} + + +class CollectorService: + def __init__(self, config: Config): + self.conf = config + self.common_db = CommonDbClient(self.conf) + self.queue = multiprocessing.Queue() + + def _collect_vim_metrics(self, vnfr: dict, vim_account_id: str): + # TODO(diazb) Add support for aws + vim_type = CollectorUtils.get_vim_type(vim_account_id) + if vim_type in VIM_COLLECTORS: + collector = VIM_COLLECTORS[vim_type](self.conf, vim_account_id) + metrics = collector.collect(vnfr) + for metric in metrics: + self.queue.put(metric) + else: + log.debug("vimtype %s is not supported.", vim_type) + + def _collect_vim_infra_metrics(self, vim_account_id: str): + vim_type = CollectorUtils.get_vim_type(vim_account_id) + if vim_type in VIM_INFRA_COLLECTORS: + collector = VIM_INFRA_COLLECTORS[vim_type](self.conf, vim_account_id) + metrics = collector.collect() + for metric in metrics: + self.queue.put(metric) + else: + log.debug("vimtype %s is not supported.", vim_type) + + def _collect_sdnc_infra_metrics(self, sdnc_id: str): + common_db = CommonDbClient(self.conf) + sdn_type = common_db.get_sdnc(sdnc_id)['type'] + if sdn_type in SDN_INFRA_COLLECTORS: + collector = SDN_INFRA_COLLECTORS[sdn_type](self.conf, sdnc_id) + metrics = collector.collect() + for metric in metrics: + self.queue.put(metric) + else: + log.debug("sdn_type %s is not supported.", sdn_type) + + def _collect_vca_metrics(self, vnfr: dict): + log.debug('_collect_vca_metrics') + log.debug('vnfr: %s', vnfr) + vca_collector = VCACollector(self.conf) + metrics = vca_collector.collect(vnfr) + for metric in metrics: + self.queue.put(metric) + + def collect_metrics(self) -> List[Metric]: + vnfrs = self.common_db.get_vnfrs() + processes = [] + for vnfr in vnfrs: + nsr_id = vnfr['nsr-id-ref'] + vnf_member_index = vnfr['member-vnf-index-ref'] + vim_account_id = self.common_db.get_vim_account_id(nsr_id, vnf_member_index) + p = multiprocessing.Process(target=self._collect_vim_metrics, + args=(vnfr, vim_account_id)) + processes.append(p) + p.start() + p = multiprocessing.Process(target=self._collect_vca_metrics, + args=(vnfr,)) + processes.append(p) + p.start() + vims = self.common_db.get_vim_accounts() + for vim in vims: + p = multiprocessing.Process(target=self._collect_vim_infra_metrics, + args=(vim['_id'],)) + processes.append(p) + p.start() + sdncs = self.common_db.get_sdncs() + for sdnc in sdncs: + p = multiprocessing.Process(target=self._collect_sdnc_infra_metrics, + args=(sdnc['_id'],)) + processes.append(p) + p.start() + for process in processes: + process.join(timeout=10) + metrics = [] + while not self.queue.empty(): + metrics.append(self.queue.get()) + return metrics diff --git a/osm_mon/collector/utils.py b/osm_mon/collector/utils.py new file mode 100644 index 0000000..272426f --- /dev/null +++ b/osm_mon/collector/utils.py @@ -0,0 +1,53 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json + +from osm_mon.collector.metric import Metric +from osm_mon.core import database +from osm_mon.core.database import VimCredentials, VimCredentialsRepository + + +class CollectorUtils(Metric): + + @staticmethod + def get_vim_type(vim_account_id) -> str: + credentials = CollectorUtils.get_credentials(vim_account_id) + config = json.loads(credentials.config) + if 'vim_type' in config: + vim_type = config['vim_type'] + return str(vim_type.lower()) + else: + return str(credentials.type) + + @staticmethod + def get_credentials(vim_account_id) -> VimCredentials: + database.db.connect() + try: + with database.db.atomic(): + return VimCredentialsRepository.get(VimCredentials.uuid == vim_account_id) + finally: + database.db.close() + + @staticmethod + def is_verify_ssl(vim_credentials: VimCredentials): + vim_config = json.loads(vim_credentials.config) + return 'insecure' not in vim_config or vim_config['insecure'] is False diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py index 2005ebe..e8ee508 100644 --- a/osm_mon/collector/vnf_collectors/openstack.py +++ b/osm_mon/collector/vnf_collectors/openstack.py @@ -25,16 +25,16 @@ import logging from typing import List import gnocchiclient.exceptions -from gnocchiclient.v1 import client as gnocchi_client from ceilometerclient.v2 import client as ceilometer_client +from gnocchiclient.v1 import client as gnocchi_client from keystoneauth1 import session from keystoneauth1.exceptions import EndpointNotFound from keystoneauth1.identity import v3 from osm_mon.collector.metric import Metric +from osm_mon.collector.utils import CollectorUtils from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector from osm_mon.collector.vnf_metric import VnfMetric -from osm_mon.core.auth import AuthManager from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config @@ -61,18 +61,17 @@ class OpenstackCollector(BaseVimCollector): super().__init__(config, vim_account_id) self.conf = config self.common_db = CommonDbClient(config) - self.auth_manager = AuthManager(config) - self.granularity = self._get_granularity(vim_account_id) self.backend = self._get_backend(vim_account_id) self.client = self._build_client(vim_account_id) + self.granularity = self._get_granularity(vim_account_id) def _get_resource_uuid(self, nsr_id, vnf_member_index, vdur_name) -> str: vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name) return vdur['vim-id'] def _build_gnocchi_client(self, vim_account_id: str) -> gnocchi_client.Client: - creds = self.auth_manager.get_credentials(vim_account_id) - verify_ssl = self.auth_manager.is_verify_ssl(vim_account_id) + creds = CollectorUtils.get_credentials(vim_account_id) + verify_ssl = CollectorUtils.is_verify_ssl(creds) auth = v3.Password(auth_url=creds.url, username=creds.user, password=creds.password, @@ -83,8 +82,8 @@ class OpenstackCollector(BaseVimCollector): return gnocchi_client.Client(session=sess) def _build_ceilometer_client(self, vim_account_id: str) -> ceilometer_client.Client: - creds = self.auth_manager.get_credentials(vim_account_id) - verify_ssl = self.auth_manager.is_verify_ssl(vim_account_id) + creds = CollectorUtils.get_credentials(vim_account_id) + verify_ssl = CollectorUtils.is_verify_ssl(creds) auth = v3.Password(auth_url=creds.url, username=creds.user, password=creds.password, @@ -94,8 +93,8 @@ class OpenstackCollector(BaseVimCollector): sess = session.Session(auth=auth, verify=verify_ssl) return ceilometer_client.Client(session=sess) - def _get_granularity(self, vim_account_id: str): - creds = self.auth_manager.get_credentials(vim_account_id) + def _get_granularity(self, vim_account_id): + creds = CollectorUtils.get_credentials(vim_account_id) vim_config = json.loads(creds.config) if 'granularity' in vim_config: return int(vim_config['granularity']) @@ -133,11 +132,11 @@ class OpenstackCollector(BaseVimCollector): metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], metric_name, measures[0].counter_volume) metrics.append(metric) - elif self.backend == 'gnocchi': + if self.backend == 'gnocchi': delta = 10 * self.granularity start_date = datetime.datetime.now() - datetime.timedelta(seconds=delta) if metric_name in INTERFACE_METRICS: - total_measure = 0.0 + total_measure = None interfaces = self.client.resource.search(resource_type='instance_network_interface', query={'=': {'instance_id': resource_id}}) for interface in interfaces: @@ -147,14 +146,17 @@ class OpenstackCollector(BaseVimCollector): resource_id=interface['id'], granularity=self.granularity) if measures: + if not total_measure: + total_measure = 0.0 total_measure += measures[-1][2] except gnocchiclient.exceptions.NotFound as e: log.debug("No metric %s found for interface %s: %s", openstack_metric_name, interface['id'], e) - metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], metric_name, - total_measure) - metrics.append(metric) + if total_measure: + metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], metric_name, + total_measure) + metrics.append(metric) else: try: measures = self.client.metric.get_measures(openstack_metric_name, @@ -170,7 +172,7 @@ class OpenstackCollector(BaseVimCollector): e) else: - raise Exception('Unknown metric backend: %s', self.backend) + raise Exception('Unknown client class: %s', self.client) return metrics def _build_client(self, vim_account_id): diff --git a/osm_mon/collector/vnf_collectors/vio.py b/osm_mon/collector/vnf_collectors/vio.py index 5bcf66e..88d3378 100644 --- a/osm_mon/collector/vnf_collectors/vio.py +++ b/osm_mon/collector/vnf_collectors/vio.py @@ -23,21 +23,22 @@ import json import logging - -from keystoneauth1 import session -from keystoneauth1.identity import v3 -from novaclient import client as nClient import re -import requests import time import traceback + +import requests import six +from keystoneauth1 import session +from keystoneauth1.identity import v3 +from novaclient import client as nClient +from osm_mon.collector.utils import CollectorUtils from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector from osm_mon.collector.vnf_metric import VnfMetric -from osm_mon.core.auth import AuthManager from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config +from osm_mon.core.database import VimCredentialsRepository, VimCredentials log = logging.getLogger(__name__) @@ -66,8 +67,7 @@ class VIOCollector(BaseVimCollector): def __init__(self, config: Config, vim_account_id: str): super().__init__(config, vim_account_id) self.common_db = CommonDbClient(config) - self.auth_manager = AuthManager(config) - vim_account_info = self.auth_manager.get_credentials(vim_account_id) + vim_account_info = CollectorUtils.get_credentials(vim_account_id) cfg = json.loads(vim_account_info.config) self.vrops_site = cfg['vrops_site'] self.vrops_user = cfg['vrops_user'] @@ -75,8 +75,8 @@ class VIOCollector(BaseVimCollector): self.client = self.connect_client(vim_account_id) def connect_client(self, vim_account_id: str): - vim_account_details = self.auth_manager.get_credentials(vim_account_id) - verify_ssl = self.auth_manager.is_verify_ssl(vim_account_id) + vim_account_details = VimCredentialsRepository.get(VimCredentials.uuid == vim_account_id) + verify_ssl = CollectorUtils.is_verify_ssl(vim_account_details) auth = v3.Password(auth_url=vim_account_details.url, username=vim_account_details.user, password=vim_account_details.password, @@ -166,7 +166,7 @@ class VIOCollector(BaseVimCollector): begin_time = end_time - time_diff api_url = "/suite-api/api/resources/{}/stats?statKey={}&begin={}&end={}".format( - resource_id, vrops_metric_name, str(begin_time), str(end_time)) + resource_id, vrops_metric_name, str(begin_time), str(end_time)) headers = {'Accept': 'application/json'} diff --git a/osm_mon/collector/vnf_collectors/vmware.py b/osm_mon/collector/vnf_collectors/vmware.py index 557fa38..0cb1e66 100644 --- a/osm_mon/collector/vnf_collectors/vmware.py +++ b/osm_mon/collector/vnf_collectors/vmware.py @@ -32,9 +32,9 @@ import six from pyvcloud.vcd.client import BasicLoginCredentials from pyvcloud.vcd.client import Client +from osm_mon.collector.utils import CollectorUtils from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector from osm_mon.collector.vnf_metric import VnfMetric -from osm_mon.core.auth import AuthManager from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config @@ -67,7 +67,6 @@ class VMwareCollector(BaseVimCollector): def __init__(self, config: Config, vim_account_id: str): super().__init__(config, vim_account_id) self.common_db = CommonDbClient(config) - self.auth_manager = AuthManager(config) vim_account = self.get_vim_account(vim_account_id) self.vrops_site = vim_account['vrops_site'] self.vrops_user = vim_account['vrops_user'] @@ -113,7 +112,7 @@ class VMwareCollector(BaseVimCollector): return - dict with vim account details """ vim_account = {} - vim_account_info = self.auth_manager.get_credentials(vim_account_id) + vim_account_info = CollectorUtils.get_credentials(vim_account_id) vim_account['name'] = vim_account_info.name vim_account['vim_tenant_name'] = vim_account_info.tenant_name diff --git a/osm_mon/core/auth.py b/osm_mon/core/auth.py deleted file mode 100644 index 4627a30..0000000 --- a/osm_mon/core/auth.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## - -import json -import logging - -from osm_mon.core.config import Config - -from osm_mon.core.database import VimCredentials, DatabaseManager - -log = logging.getLogger(__name__) - - -class AuthManager: - def __init__(self, config: Config): - self.database_manager = DatabaseManager(config) - - def store_auth_credentials(self, creds_dict): - log.info(creds_dict) - credentials = VimCredentials() - credentials.uuid = creds_dict['_id'] - credentials.name = creds_dict['name'] - credentials.type = creds_dict['vim_type'] - credentials.url = creds_dict['vim_url'] - credentials.user = creds_dict['vim_user'] - credentials.password = creds_dict['vim_password'] - credentials.tenant_name = creds_dict['vim_tenant_name'] - if 'config' not in creds_dict: - creds_dict['config'] = {} - credentials.config = json.dumps(creds_dict['config']) - self.database_manager.save_credentials(credentials) - - def get_credentials(self, vim_uuid): - creds = self.database_manager.get_credentials(vim_uuid) - return creds - - def delete_auth_credentials(self, creds_dict): - credentials = self.get_credentials(creds_dict['_id']) - if credentials: - credentials.delete_instance() - - def get_config(self, vim_uuid): - return json.loads(self.get_credentials(vim_uuid).config) - - def is_verify_ssl(self, vim_uuid): - vim_config = self.get_config(vim_uuid) - return 'insecure' not in vim_config or vim_config['insecure'] is False diff --git a/osm_mon/core/database.py b/osm_mon/core/database.py index d1c2e6b..2f51b1e 100644 --- a/osm_mon/core/database.py +++ b/osm_mon/core/database.py @@ -22,10 +22,9 @@ # contact: bdiaz@whitestack.com or glavado@whitestack.com ## -import json import logging import os -import uuid +from typing import Iterable from peewee import CharField, TextField, FloatField, Model, AutoField, Proxy from peewee_migrate import Router @@ -81,77 +80,33 @@ class DatabaseManager: router.run() db.close() - def get_credentials(self, vim_uuid: str = None) -> VimCredentials: - db.connect() - try: - with db.atomic(): - vim_credentials = VimCredentials.get_or_none(VimCredentials.uuid == vim_uuid) - return vim_credentials - finally: - db.close() - - def save_credentials(self, vim_credentials) -> VimCredentials: - """Saves vim credentials. If a record with same uuid exists, overwrite it.""" - db.connect() - try: - with db.atomic(): - exists = VimCredentials.get_or_none(VimCredentials.uuid == vim_credentials.uuid) - if exists: - vim_credentials.id = exists.id - vim_credentials.save() - return vim_credentials - finally: - db.close() - - def get_alarm(self, alarm_id) -> Alarm: - db.connect() - try: - with db.atomic(): - alarm = (Alarm.select() - .where(Alarm.alarm_id == alarm_id) - .get()) - return alarm - finally: - db.close() - - def save_alarm(self, name, threshold, operation, severity, statistic, metric_name, vdur_name, - vnf_member_index, nsr_id) -> Alarm: - """Saves alarm.""" - # TODO: Add uuid optional param and check if exists to handle updates (see self.save_credentials) - db.connect() - try: - with db.atomic(): - alarm = Alarm() - alarm.uuid = str(uuid.uuid4()) - alarm.name = name - alarm.threshold = threshold - alarm.operation = operation - alarm.severity = severity - alarm.statistic = statistic - alarm.monitoring_param = metric_name - alarm.vdur_name = vdur_name - alarm.vnf_member_index = vnf_member_index - alarm.nsr_id = nsr_id - alarm.save() - return alarm - finally: - db.close() - - def delete_alarm(self, alarm_uuid) -> None: - db.connect() - with db.atomic(): - alarm = (Alarm.select() - .where(Alarm.uuid == alarm_uuid) - .get()) - alarm.delete_instance() - db.close() - def get_vim_type(self, vim_account_id) -> str: - """Get the vim type that is required by the message.""" - credentials = self.get_credentials(vim_account_id) - config = json.loads(credentials.config) - if 'vim_type' in config: - vim_type = config['vim_type'] - return str(vim_type.lower()) +class VimCredentialsRepository: + @staticmethod + def upsert(**query) -> VimCredentials: + vim_credentials = VimCredentials.get_or_none(**query) + if vim_credentials: + query.update({'id': vim_credentials.id}) + vim_id = VimCredentials.insert(**query).on_conflict_replace().execute() + return VimCredentials.get(id=vim_id) + + @staticmethod + def get(*expressions) -> VimCredentials: + return VimCredentials.select().where(*expressions).get() + + +class AlarmRepository: + @staticmethod + def create(**query) -> Alarm: + return Alarm.create(**query) + + @staticmethod + def get(*expressions) -> Alarm: + return Alarm.select().where(*expressions).get() + + @staticmethod + def list(*expressions) -> Iterable[Alarm]: + if expressions == (): + return Alarm.select() else: - return str(credentials.type) + return Alarm.select().where(*expressions) diff --git a/osm_mon/core/mon.yaml b/osm_mon/core/mon.yaml index b1607ec..19e8f3a 100644 --- a/osm_mon/core/mon.yaml +++ b/osm_mon/core/mon.yaml @@ -44,6 +44,7 @@ collector: evaluator: interval: 30 + backend: prometheus prometheus: url: http://prometheus:9090 diff --git a/osm_mon/evaluator/backends/__init__.py b/osm_mon/evaluator/backends/__init__.py new file mode 100644 index 0000000..4450364 --- /dev/null +++ b/osm_mon/evaluator/backends/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## diff --git a/osm_mon/evaluator/backends/base.py b/osm_mon/evaluator/backends/base.py new file mode 100644 index 0000000..0e9fc0d --- /dev/null +++ b/osm_mon/evaluator/backends/base.py @@ -0,0 +1,30 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +from osm_mon.core.config import Config + + +class BaseBackend: + def __init__(self, config: Config): + pass + + def get_metric_value(self, metric_name, nsr_id, vdur_name, vnf_member_index): + pass diff --git a/osm_mon/evaluator/backends/prometheus.py b/osm_mon/evaluator/backends/prometheus.py new file mode 100644 index 0000000..9ff50d6 --- /dev/null +++ b/osm_mon/evaluator/backends/prometheus.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import logging + +import requests + +from osm_mon.core.config import Config +from osm_mon.evaluator.backends.base import BaseBackend + +log = logging.getLogger(__name__) + +OSM_METRIC_PREFIX = 'osm_' + + +class PrometheusBackend(BaseBackend): + + def __init__(self, config: Config): + super().__init__(config) + self.conf = config + + def get_metric_value(self, metric_name, nsr_id, vdur_name, vnf_member_index): + query_section = "query={0}{{ns_id=\"{1}\",vdu_name=\"{2}\",vnf_member_index=\"{3}\"}}".format( + OSM_METRIC_PREFIX + metric_name, nsr_id, vdur_name, vnf_member_index) + request_url = self.conf.get('prometheus', 'url') + "/api/v1/query?" + query_section + log.info("Querying Prometheus: %s", request_url) + r = requests.get(request_url, timeout=int(self.conf.get('global', 'request_timeout'))) + if r.status_code == 200: + json_response = r.json() + if json_response['status'] == 'success': + result = json_response['data']['result'] + if len(result): + metric_value = float(result[0]['value'][1]) + log.info("Metric value: %s", metric_value) + return metric_value + else: + return None + else: + log.warning("Prometheus response is not success. Got status %s", json_response['status']) + else: + log.warning("Error contacting Prometheus. Got status code %s: %s", r.status_code, r.text) + return None diff --git a/osm_mon/evaluator/evaluator.py b/osm_mon/evaluator/evaluator.py index 24e8e43..d3fdfd5 100644 --- a/osm_mon/evaluator/evaluator.py +++ b/osm_mon/evaluator/evaluator.py @@ -24,28 +24,18 @@ import asyncio import logging import multiprocessing import time -from enum import Enum import peewee -import requests -from osm_common.dbbase import DbException -from osm_mon.collector.backends.prometheus import OSM_METRIC_PREFIX -from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config -from osm_mon.core.database import DatabaseManager, Alarm +from osm_mon.core.database import Alarm from osm_mon.core.message_bus_client import MessageBusClient from osm_mon.core.response import ResponseBuilder +from osm_mon.evaluator.service import EvaluatorService, AlarmStatus log = logging.getLogger(__name__) -class AlarmStatus(Enum): - ALARM = 'alarm' - OK = 'ok' - INSUFFICIENT = 'insufficient-data' - - class Evaluator: def __init__(self, config: Config, loop=None): @@ -53,52 +43,9 @@ class Evaluator: if not loop: loop = asyncio.get_event_loop() self.loop = loop - self.common_db = CommonDbClient(self.conf) - self.plugins = [] - self.database_manager = DatabaseManager(self.conf) - self.database_manager.create_tables() - self.queue = multiprocessing.Queue() + self.service = EvaluatorService(config) self.msg_bus = MessageBusClient(config) - def _evaluate_metric(self, - nsr_id: str, - vnf_member_index: int, - vdur_name: str, - metric_name: str, - alarm: Alarm): - log.debug("_evaluate_metric") - # TODO: Refactor to fit backend plugin model - query_section = "query={0}{{ns_id=\"{1}\",vdu_name=\"{2}\",vnf_member_index=\"{3}\"}}".format( - OSM_METRIC_PREFIX + metric_name, nsr_id, vdur_name, vnf_member_index) - request_url = self.conf.get('prometheus', 'url') + "/api/v1/query?" + query_section - log.info("Querying Prometheus: %s", request_url) - r = requests.get(request_url, timeout=int(self.conf.get('global', 'request_timeout'))) - if r.status_code == 200: - json_response = r.json() - if json_response['status'] == 'success': - result = json_response['data']['result'] - if result: - metric_value = float(result[0]['value'][1]) - log.info("Metric value: %s", metric_value) - if alarm.operation.upper() == 'GT': - if metric_value > alarm.threshold: - self.queue.put((alarm, AlarmStatus.ALARM)) - else: - self.queue.put((alarm, AlarmStatus.OK)) - elif alarm.operation.upper() == 'LT': - if metric_value < alarm.threshold: - self.queue.put((alarm, AlarmStatus.ALARM)) - else: - self.queue.put((alarm, AlarmStatus.OK)) - else: - log.warning("No metric result for alarm %s", alarm.id) - self.queue.put((alarm, AlarmStatus.INSUFFICIENT)) - - else: - log.warning("Prometheus response is not success. Got status %s", json_response['status']) - else: - log.warning("Error contacting Prometheus. Got status code %s: %s", r.status_code, r.text) - def evaluate_forever(self): log.debug('evaluate_forever') while True: @@ -113,65 +60,7 @@ class Evaluator: def evaluate(self): log.debug('evaluate') - processes = [] - for alarm in Alarm.select(): - try: - vnfr = self.common_db.get_vnfr(alarm.nsr_id, alarm.vnf_member_index) - except DbException: - log.exception("Error getting vnfr: ") - continue - vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) - try: - vdur = next(filter(lambda vdur: vdur['name'] == alarm.vdur_name, vnfr['vdur'])) - except StopIteration: - log.warning("No vdur found with name %s for alarm %s", alarm.vdur_name, alarm.id) - continue - vdu = next(filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])) - vnf_monitoring_param = next( - filter(lambda param: param['id'] == alarm.monitoring_param, vnfd['monitoring-param'])) - nsr_id = vnfr['nsr-id-ref'] - vnf_member_index = vnfr['member-vnf-index-ref'] - vdur_name = vdur['name'] - if 'vdu-monitoring-param' in vnf_monitoring_param: - vdu_monitoring_param = next(filter( - lambda param: param['id'] == vnf_monitoring_param['vdu-monitoring-param'][ - 'vdu-monitoring-param-ref'], vdu['monitoring-param'])) - nfvi_metric = vdu_monitoring_param['nfvi-metric'] - - p = multiprocessing.Process(target=self._evaluate_metric, - args=(nsr_id, - vnf_member_index, - vdur_name, - nfvi_metric, - alarm)) - processes.append(p) - p.start() - if 'vdu-metric' in vnf_monitoring_param: - vnf_metric_name = vnf_monitoring_param['vdu-metric']['vdu-metric-name-ref'] - p = multiprocessing.Process(target=self._evaluate_metric, - args=(nsr_id, - vnf_member_index, - vdur_name, - vnf_metric_name, - alarm)) - processes.append(p) - p.start() - if 'vnf-metric' in vnf_monitoring_param: - vnf_metric_name = vnf_monitoring_param['vnf-metric']['vnf-metric-name-ref'] - p = multiprocessing.Process(target=self._evaluate_metric, - args=(nsr_id, - vnf_member_index, - '', - vnf_metric_name, - alarm)) - processes.append(p) - p.start() - - for process in processes: - process.join(timeout=10) - alarms_tuples = [] - while not self.queue.empty(): - alarms_tuples.append(self.queue.get()) + alarms_tuples = self.service.evaluate_alarms() for alarm, status in alarms_tuples: p = multiprocessing.Process(target=self.notify_alarm, args=(alarm, status)) diff --git a/osm_mon/evaluator/service.py b/osm_mon/evaluator/service.py new file mode 100644 index 0000000..0868e32 --- /dev/null +++ b/osm_mon/evaluator/service.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import logging +import multiprocessing +from enum import Enum +from typing import Tuple, List + +from osm_common.dbbase import DbException + +from osm_mon.core import database +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +from osm_mon.core.database import Alarm, AlarmRepository +from osm_mon.evaluator.backends.prometheus import PrometheusBackend + +log = logging.getLogger(__name__) + +BACKENDS = { + 'prometheus': PrometheusBackend +} + + +class AlarmStatus(Enum): + ALARM = 'alarm' + OK = 'ok' + INSUFFICIENT = 'insufficient-data' + + +class EvaluatorService: + + def __init__(self, config: Config): + self.conf = config + self.common_db = CommonDbClient(self.conf) + self.queue = multiprocessing.Queue() + + def _get_metric_value(self, + nsr_id: str, + vnf_member_index: int, + vdur_name: str, + metric_name: str): + return BACKENDS[self.conf.get('evaluator', 'backend')]().get_metric_value(metric_name, nsr_id, vdur_name, + vnf_member_index) + + def _evaluate_metric(self, + nsr_id: str, + vnf_member_index: int, + vdur_name: str, + metric_name: str, + alarm: Alarm): + log.debug("_evaluate_metric") + metric_value = self._get_metric_value(nsr_id, vnf_member_index, vdur_name, metric_name) + if not metric_value: + log.warning("No metric result for alarm %s", alarm.id) + self.queue.put((alarm, AlarmStatus.INSUFFICIENT)) + else: + if alarm.operation.upper() == 'GT': + if metric_value > alarm.threshold: + self.queue.put((alarm, AlarmStatus.ALARM)) + else: + self.queue.put((alarm, AlarmStatus.OK)) + elif alarm.operation.upper() == 'LT': + if metric_value < alarm.threshold: + self.queue.put((alarm, AlarmStatus.ALARM)) + else: + self.queue.put((alarm, AlarmStatus.OK)) + + def evaluate_alarms(self) -> List[Tuple[Alarm, AlarmStatus]]: + log.debug('evaluate_alarms') + processes = [] + database.db.connect() + try: + with database.db.atomic(): + for alarm in AlarmRepository.list(): + try: + vnfr = self.common_db.get_vnfr(alarm.nsr_id, alarm.vnf_member_index) + except DbException: + log.exception("Error getting vnfr: ") + continue + vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) + try: + vdur = next(filter(lambda vdur: vdur['name'] == alarm.vdur_name, vnfr['vdur'])) + except StopIteration: + log.warning("No vdur found with name %s for alarm %s", alarm.vdur_name, alarm.id) + continue + vdu = next(filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])) + vnf_monitoring_param = next( + filter(lambda param: param['id'] == alarm.monitoring_param, vnfd['monitoring-param'])) + nsr_id = vnfr['nsr-id-ref'] + vnf_member_index = vnfr['member-vnf-index-ref'] + vdur_name = vdur['name'] + if 'vdu-monitoring-param' in vnf_monitoring_param: + vdu_monitoring_param = next(filter( + lambda param: param['id'] == vnf_monitoring_param['vdu-monitoring-param'][ + 'vdu-monitoring-param-ref'], vdu['monitoring-param'])) + nfvi_metric = vdu_monitoring_param['nfvi-metric'] + + p = multiprocessing.Process(target=self._evaluate_metric, + args=(nsr_id, + vnf_member_index, + vdur_name, + nfvi_metric, + alarm)) + processes.append(p) + p.start() + if 'vdu-metric' in vnf_monitoring_param: + vnf_metric_name = vnf_monitoring_param['vdu-metric']['vdu-metric-name-ref'] + p = multiprocessing.Process(target=self._evaluate_metric, + args=(nsr_id, + vnf_member_index, + vdur_name, + vnf_metric_name, + alarm)) + processes.append(p) + p.start() + if 'vnf-metric' in vnf_monitoring_param: + vnf_metric_name = vnf_monitoring_param['vnf-metric']['vnf-metric-name-ref'] + p = multiprocessing.Process(target=self._evaluate_metric, + args=(nsr_id, + vnf_member_index, + '', + vnf_metric_name, + alarm)) + processes.append(p) + p.start() + + for process in processes: + process.join(timeout=10) + alarms_tuples = [] + while not self.queue.empty(): + alarms_tuples.append(self.queue.get()) + return alarms_tuples + finally: + database.db.close() diff --git a/osm_mon/server/server.py b/osm_mon/server/server.py index 0011618..1e3e7f8 100755 --- a/osm_mon/server/server.py +++ b/osm_mon/server/server.py @@ -20,17 +20,17 @@ # For those usages not covered by the Apache License, Version 2.0 please # contact: bdiaz@whitestack.com or glavado@whitestack.com ## -"""A common KafkaConsumer for all MON plugins.""" +""" +MON component in charge of CRUD operations for vim_accounts and alarms. It uses the message bus to communicate. +""" import asyncio import json import logging -from osm_mon.core.auth import AuthManager -from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config -from osm_mon.core.database import DatabaseManager from osm_mon.core.message_bus_client import MessageBusClient from osm_mon.core.response import ResponseBuilder +from osm_mon.server.service import ServerService log = logging.getLogger(__name__) @@ -42,11 +42,8 @@ class Server: if not loop: loop = asyncio.get_event_loop() self.loop = loop - self.auth_manager = AuthManager(config) - self.database_manager = DatabaseManager(config) - self.database_manager.create_tables() - self.common_db = CommonDbClient(config) self.msg_bus = MessageBusClient(config) + self.service = ServerService(config) def run(self): self.loop.run_until_complete(self.start()) @@ -63,21 +60,20 @@ class Server: try: if topic == "vim_account": if key == "create" or key == "edit": - values['vim_password'] = self.common_db.decrypt_vim_password(values['vim_password'], - values['schema_version'], - values['_id']) - - vim_config_encrypted = ("admin_password", "nsx_password", "vcenter_password") - if 'config' in values: - for key in values['config']: - if key in vim_config_encrypted: - values['config'][key] = self.common_db.decrypt_vim_password(values['config'][key], - values['schema_version'], - values['_id']) - self.auth_manager.store_auth_credentials(values) + if 'config' not in values: + values['config'] = {} + self.service.upsert_vim_account(values['_id'], + values['name'], + values['vim_type'], + values['vim_url'], + values['vim_user'], + values['vim_password'], + values['vim_tenant_name'], + values['schema_version'], + values['config']) if key == "delete": - self.auth_manager.delete_auth_credentials(values) + self.service.delete_vim_account(values['_id']) elif topic == "alarm_request": if key == "create_alarm_request": @@ -85,7 +81,7 @@ class Server: cor_id = alarm_details['correlation_id'] response_builder = ResponseBuilder() try: - alarm = self.database_manager.save_alarm( + alarm = self.service.create_alarm( alarm_details['alarm_name'], alarm_details['threshold_value'], alarm_details['operation'].lower(), @@ -114,7 +110,7 @@ class Server: response_builder = ResponseBuilder() cor_id = alarm_details['correlation_id'] try: - self.database_manager.delete_alarm(alarm_uuid) + self.service.delete_alarm(alarm_uuid) response = response_builder.generate_response('delete_alarm_response', cor_id=cor_id, status=True, diff --git a/osm_mon/server/service.py b/osm_mon/server/service.py new file mode 100755 index 0000000..43a0d8e --- /dev/null +++ b/osm_mon/server/service.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +import logging +import uuid + +from osm_mon.core import database +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +from osm_mon.core.database import VimCredentialsRepository, VimCredentials, AlarmRepository, Alarm + +log = logging.getLogger(__name__) + + +class ServerService: + + def __init__(self, config: Config): + self.common_db = CommonDbClient(config) + + def upsert_vim_account(self, + vim_uuid: str, + name: str, + vim_type: str, + url: str, + user: str, + password: str, + tenant_name: str, + schema_version: str, + config: dict) -> VimCredentials: + decrypted_vim_password = self.common_db.decrypt_vim_password(password, + schema_version, + vim_uuid) + + vim_config_encrypted = ("admin_password", "nsx_password", "vcenter_password") + for key in config: + if key in vim_config_encrypted: + config[key] = self.common_db.decrypt_vim_password(config[key], + schema_version, + vim_uuid) + database.db.connect() + try: + with database.db.atomic(): + return VimCredentialsRepository.upsert( + uuid=vim_uuid, + name=name, + type=vim_type, + url=url, + user=user, + password=decrypted_vim_password, + tenant_name=tenant_name, + config=json.dumps(config) + ) + finally: + database.db.close() + + def delete_vim_account(self, vim_uuid: str) -> None: + database.db.connect() + try: + with database.db.atomic(): + vim_credentials = VimCredentialsRepository.get(VimCredentials.uuid == vim_uuid) + vim_credentials.delete_instance() + finally: + database.db.close() + + def create_alarm(self, + name: str, + threshold: str, + operation: str, + severity: str, + statistic: str, + metric_name: str, + vdur_name: str, + vnf_member_index: str, + nsr_id: str) -> Alarm: + database.db.connect() + try: + with database.db.atomic(): + return AlarmRepository.create( + uuid=str(uuid.uuid4()), + name=name, + threshold=threshold, + operation=operation.lower(), + severity=severity.lower(), + statistic=statistic.lower(), + monitoring_param=metric_name, + vdur_name=vdur_name, + vnf_member_index=vnf_member_index, + nsr_id=nsr_id + ) + + finally: + database.db.close() + + def delete_alarm(self, + alarm_uuid: str) -> None: + database.db.connect() + try: + with database.db.atomic(): + alarm = AlarmRepository.get(Alarm.uuid == alarm_uuid) + alarm.delete_instance() + finally: + database.db.close() diff --git a/osm_mon/tests/collector/__init__.py b/osm_mon/tests/collector/__init__.py deleted file mode 100644 index 4450364..0000000 --- a/osm_mon/tests/collector/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## diff --git a/osm_mon/tests/collector/test_collector.py b/osm_mon/tests/collector/test_collector.py deleted file mode 100644 index 4bbe10e..0000000 --- a/osm_mon/tests/collector/test_collector.py +++ /dev/null @@ -1,72 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## -import os -import unittest -from unittest import mock - -from osm_mon.collector.collector import Collector -from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector -from osm_mon.core.config import Config -from osm_mon.core.database import DatabaseManager, db - - -class CollectorTest(unittest.TestCase): - def setUp(self): - super().setUp() - os.environ["OSMMON_SQL_DATABASE_URI"] = "sqlite:///:memory:" - self.config = Config() - db_manager = DatabaseManager(self.config) - db_manager.create_tables() - - def tearDown(self): - super().tearDown() - db.close() - - @mock.patch("osm_mon.collector.collector.CommonDbClient", mock.Mock()) - @mock.patch.object(Collector, "_init_backends", mock.Mock()) - @mock.patch.object(OpenstackCollector, "__init__", lambda *args, **kwargs: None) - @mock.patch.object(OpenstackCollector, "collect") - @mock.patch.object(DatabaseManager, "get_vim_type") - def test_init_vim_collector_and_collect_openstack(self, _get_vim_type, collect): - _get_vim_type.return_value = 'openstack' - collector = Collector(self.config) - collector._collect_vim_metrics({}, 'test_vim_account_id') - collect.assert_called_once_with({}) - - @mock.patch("osm_mon.collector.collector.CommonDbClient", mock.Mock()) - @mock.patch.object(Collector, "_init_backends", mock.Mock()) - @mock.patch.object(OpenstackCollector, "collect") - @mock.patch.object(DatabaseManager, "get_vim_type") - def test_init_vim_collector_and_collect_unknown(self, _get_vim_type, openstack_collect): - _get_vim_type.return_value = 'unknown' - collector = Collector(self.config) - collector._collect_vim_metrics({}, 'test_vim_account_id') - openstack_collect.assert_not_called() - - @mock.patch("osm_mon.collector.collector.CommonDbClient", mock.Mock()) - @mock.patch("osm_mon.collector.collector.VCACollector", autospec=True) - def test_collect_vca_metrics(self, vca_collector): - collector = Collector(self.config) - collector._collect_vca_metrics({}) - vca_collector.assert_called_once_with(self.config) - vca_collector.return_value.collect.assert_called_once_with({}) diff --git a/osm_mon/tests/core/__init__.py b/osm_mon/tests/core/__init__.py deleted file mode 100644 index ce869a3..0000000 --- a/osm_mon/tests/core/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# Copyright 2017 Intel Research and Development Ireland Limited -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Intel Corporation - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com diff --git a/osm_mon/tests/core/test_common_db_client.py b/osm_mon/tests/core/test_common_db_client.py deleted file mode 100644 index 81159ee..0000000 --- a/osm_mon/tests/core/test_common_db_client.py +++ /dev/null @@ -1,99 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## -import unittest -from unittest import mock - -from osm_common import dbmongo - -from osm_mon.core.common_db import CommonDbClient -from osm_mon.core.config import Config - - -class CommonDbClientTest(unittest.TestCase): - def setUp(self): - super().setUp() - self.config = Config() - - @mock.patch.object(dbmongo.DbMongo, "db_connect", mock.Mock()) - @mock.patch.object(CommonDbClient, "get_vnfr") - def test_get_vim_id(self, get_vnfr): - get_vnfr.return_value = {'_id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01', - '_admin': { - 'projects_read': ['admin'], 'created': 1526044312.102287, - 'modified': 1526044312.102287, 'projects_write': ['admin'] - }, - 'vim-account-id': 'c1740601-7287-48c8-a2c9-bce8fee459eb', - 'nsr-id-ref': '5ec3f571-d540-4cb0-9992-971d1b08312e', - 'vdur': [ - { - 'internal-connection-point': [], - 'vdu-id-ref': 'ubuntuvnf_vnfd-VM', - 'id': 'ffd73f33-c8bb-4541-a977-44dcc3cbe28d', - 'vim-id': '27042672-5190-4209-b844-95bbaeea7ea7', - 'name': 'ubuntuvnf_vnfd-VM' - } - ], - 'vnfd-ref': 'ubuntuvnf_vnfd', - 'member-vnf-index-ref': '1', - 'created-time': 1526044312.0999322, - 'vnfd-id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01', - 'id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01'} - common_db_client = CommonDbClient(self.config) - vim_account_id = common_db_client.get_vim_account_id('5ec3f571-d540-4cb0-9992-971d1b08312e', 1) - self.assertEqual(vim_account_id, 'c1740601-7287-48c8-a2c9-bce8fee459eb') - - @mock.patch.object(dbmongo.DbMongo, "db_connect", mock.Mock()) - @mock.patch.object(dbmongo.DbMongo, "get_one") - def test_get_vdur(self, get_one): - get_one.return_value = {'_id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01', - '_admin': { - 'projects_read': ['admin'], 'created': 1526044312.102287, - 'modified': 1526044312.102287, 'projects_write': ['admin'] - }, - 'vim-account-id': 'c1740601-7287-48c8-a2c9-bce8fee459eb', - 'nsr-id-ref': '5ec3f571-d540-4cb0-9992-971d1b08312e', - 'vdur': [ - { - 'internal-connection-point': [], - 'vdu-id-ref': 'ubuntuvnf_vnfd-VM', - 'id': 'ffd73f33-c8bb-4541-a977-44dcc3cbe28d', - 'vim-id': '27042672-5190-4209-b844-95bbaeea7ea7', - 'name': 'ubuntuvnf_vnfd-VM' - } - ], - 'vnfd-ref': 'ubuntuvnf_vnfd', - 'member-vnf-index-ref': '1', - 'created-time': 1526044312.0999322, - 'vnfd-id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01', - 'id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01'} - common_db_client = CommonDbClient(self.config) - vdur = common_db_client.get_vdur('5ec3f571-d540-4cb0-9992-971d1b08312e', '1', 'ubuntuvnf_vnfd-VM') - expected_vdur = { - 'internal-connection-point': [], - 'vdu-id-ref': 'ubuntuvnf_vnfd-VM', - 'id': 'ffd73f33-c8bb-4541-a977-44dcc3cbe28d', - 'vim-id': '27042672-5190-4209-b844-95bbaeea7ea7', - 'name': 'ubuntuvnf_vnfd-VM' - } - - self.assertDictEqual(vdur, expected_vdur) diff --git a/osm_mon/tests/core/test_database.py b/osm_mon/tests/core/test_database.py deleted file mode 100644 index 3f8eb7e..0000000 --- a/osm_mon/tests/core/test_database.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## -import unittest -from unittest import mock - -from osm_mon.core.config import Config - -from osm_mon.core.database import VimCredentials, DatabaseManager - - -class DatabaseManagerTest(unittest.TestCase): - def setUp(self): - super().setUp() - self.config = Config() - - @mock.patch.object(DatabaseManager, "get_credentials") - def test_get_vim_type(self, get_credentials): - mock_creds = VimCredentials() - mock_creds.id = 'test_id' - mock_creds.user = 'user' - mock_creds.url = 'url' - mock_creds.password = 'password' - mock_creds.tenant_name = 'tenant_name' - mock_creds.type = 'openstack' - mock_creds.config = '{}' - - get_credentials.return_value = mock_creds - database_manager = DatabaseManager(self.config) - vim_type = database_manager.get_vim_type('test_id') - self.assertEqual(vim_type, 'openstack') diff --git a/osm_mon/tests/core/test_message_bus_client.py b/osm_mon/tests/core/test_message_bus_client.py deleted file mode 100644 index 292fbe3..0000000 --- a/osm_mon/tests/core/test_message_bus_client.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## -import asyncio -from unittest import TestCase, mock - -from osm_common.msgkafka import MsgKafka - -from osm_mon.core.message_bus_client import MessageBusClient -from osm_mon.core.config import Config - - -class TestMessageBusClient(TestCase): - - def setUp(self): - self.config = Config() - self.config.set('message', 'driver', 'kafka') - self.loop = asyncio.new_event_loop() - - @mock.patch.object(MsgKafka, 'aioread') - def test_aioread(self, aioread): - async def mock_callback(): - pass - - future = asyncio.Future(loop=self.loop) - future.set_result('mock') - aioread.return_value = future - msg_bus = MessageBusClient(self.config, loop=self.loop) - topic = 'test_topic' - self.loop.run_until_complete(msg_bus.aioread([topic], mock_callback)) - aioread.assert_called_with(['test_topic'], self.loop, aiocallback=mock_callback) - - @mock.patch.object(MsgKafka, 'aiowrite') - def test_aiowrite(self, aiowrite): - future = asyncio.Future(loop=self.loop) - future.set_result('mock') - aiowrite.return_value = future - msg_bus = MessageBusClient(self.config, loop=self.loop) - topic = 'test_topic' - key = 'test_key' - msg = {'test': 'test_msg'} - self.loop.run_until_complete(msg_bus.aiowrite(topic, key, msg)) - aiowrite.assert_called_with(topic, key, msg, self.loop) - - @mock.patch.object(MsgKafka, 'aioread') - def test_aioread_once(self, aioread): - future = asyncio.Future(loop=self.loop) - future.set_result('mock') - aioread.return_value = future - msg_bus = MessageBusClient(self.config, loop=self.loop) - topic = 'test_topic' - self.loop.run_until_complete(msg_bus.aioread_once(topic)) - aioread.assert_called_with('test_topic', self.loop) diff --git a/osm_mon/tests/unit/__init__.py b/osm_mon/tests/unit/__init__.py new file mode 100644 index 0000000..971f4e9 --- /dev/null +++ b/osm_mon/tests/unit/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## diff --git a/osm_mon/tests/unit/collector/__init__.py b/osm_mon/tests/unit/collector/__init__.py new file mode 100644 index 0000000..4450364 --- /dev/null +++ b/osm_mon/tests/unit/collector/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## diff --git a/osm_mon/tests/unit/collector/test_collector.py b/osm_mon/tests/unit/collector/test_collector.py new file mode 100644 index 0000000..4a3e859 --- /dev/null +++ b/osm_mon/tests/unit/collector/test_collector.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import unittest + +from osm_mon.core.config import Config + + +class CollectorTest(unittest.TestCase): + def setUp(self): + super().setUp() + self.config = Config() + + def tearDown(self): + super().tearDown() diff --git a/osm_mon/tests/unit/collector/test_collector_service.py b/osm_mon/tests/unit/collector/test_collector_service.py new file mode 100644 index 0000000..4dab43f --- /dev/null +++ b/osm_mon/tests/unit/collector/test_collector_service.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +from unittest import TestCase, mock + +from osm_mon.collector.service import CollectorService +from osm_mon.collector.utils import CollectorUtils +from osm_mon.collector.vnf_collectors.openstack import OpenstackCollector +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config + + +@mock.patch.object(CommonDbClient, "__init__", lambda *args, **kwargs: None) +class CollectorServiceTest(TestCase): + def setUp(self): + super().setUp() + self.config = Config() + + @mock.patch.object(OpenstackCollector, "__init__", lambda *args, **kwargs: None) + @mock.patch.object(OpenstackCollector, "collect") + @mock.patch.object(CollectorUtils, "get_vim_type") + def test_init_vim_collector_and_collect_openstack(self, _get_vim_type, collect): + _get_vim_type.return_value = 'openstack' + collector = CollectorService(self.config) + collector._collect_vim_metrics({}, 'test_vim_account_id') + collect.assert_called_once_with({}) + + @mock.patch.object(OpenstackCollector, "collect") + @mock.patch.object(CollectorUtils, "get_vim_type") + def test_init_vim_collector_and_collect_unknown(self, _get_vim_type, openstack_collect): + _get_vim_type.return_value = 'unknown' + collector = CollectorService(self.config) + collector._collect_vim_metrics({}, 'test_vim_account_id') + openstack_collect.assert_not_called() + + @mock.patch("osm_mon.collector.service.VCACollector", autospec=True) + def test_collect_vca_metrics(self, vca_collector): + collector = CollectorService(self.config) + collector._collect_vca_metrics({}) + vca_collector.assert_called_once_with(self.config) + vca_collector.return_value.collect.assert_called_once_with({}) diff --git a/osm_mon/tests/unit/collector/test_collector_utils.py b/osm_mon/tests/unit/collector/test_collector_utils.py new file mode 100644 index 0000000..a308522 --- /dev/null +++ b/osm_mon/tests/unit/collector/test_collector_utils.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +from unittest import TestCase, mock + +from osm_mon.collector.utils import CollectorUtils +from osm_mon.core.config import Config +from osm_mon.core.database import VimCredentialsRepository, VimCredentials + + +class CollectorServiceTest(TestCase): + def setUp(self): + super().setUp() + self.config = Config() + + @mock.patch.object(VimCredentialsRepository, "get") + @mock.patch('osm_mon.core.database.db') + def test_get_vim_type(self, db, get_credentials): + mock_creds = VimCredentials() + mock_creds.id = 'test_id' + mock_creds.user = 'user' + mock_creds.url = 'url' + mock_creds.password = 'password' + mock_creds.tenant_name = 'tenant_name' + mock_creds.type = 'openstack' + mock_creds.config = '{}' + + get_credentials.return_value = mock_creds + vim_type = CollectorUtils.get_vim_type('test_id') + self.assertEqual(vim_type, 'openstack') diff --git a/osm_mon/tests/unit/core/__init__.py b/osm_mon/tests/unit/core/__init__.py new file mode 100644 index 0000000..ce869a3 --- /dev/null +++ b/osm_mon/tests/unit/core/__init__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com diff --git a/osm_mon/tests/unit/core/test_common_db_client.py b/osm_mon/tests/unit/core/test_common_db_client.py new file mode 100644 index 0000000..81159ee --- /dev/null +++ b/osm_mon/tests/unit/core/test_common_db_client.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import unittest +from unittest import mock + +from osm_common import dbmongo + +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config + + +class CommonDbClientTest(unittest.TestCase): + def setUp(self): + super().setUp() + self.config = Config() + + @mock.patch.object(dbmongo.DbMongo, "db_connect", mock.Mock()) + @mock.patch.object(CommonDbClient, "get_vnfr") + def test_get_vim_id(self, get_vnfr): + get_vnfr.return_value = {'_id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01', + '_admin': { + 'projects_read': ['admin'], 'created': 1526044312.102287, + 'modified': 1526044312.102287, 'projects_write': ['admin'] + }, + 'vim-account-id': 'c1740601-7287-48c8-a2c9-bce8fee459eb', + 'nsr-id-ref': '5ec3f571-d540-4cb0-9992-971d1b08312e', + 'vdur': [ + { + 'internal-connection-point': [], + 'vdu-id-ref': 'ubuntuvnf_vnfd-VM', + 'id': 'ffd73f33-c8bb-4541-a977-44dcc3cbe28d', + 'vim-id': '27042672-5190-4209-b844-95bbaeea7ea7', + 'name': 'ubuntuvnf_vnfd-VM' + } + ], + 'vnfd-ref': 'ubuntuvnf_vnfd', + 'member-vnf-index-ref': '1', + 'created-time': 1526044312.0999322, + 'vnfd-id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01', + 'id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01'} + common_db_client = CommonDbClient(self.config) + vim_account_id = common_db_client.get_vim_account_id('5ec3f571-d540-4cb0-9992-971d1b08312e', 1) + self.assertEqual(vim_account_id, 'c1740601-7287-48c8-a2c9-bce8fee459eb') + + @mock.patch.object(dbmongo.DbMongo, "db_connect", mock.Mock()) + @mock.patch.object(dbmongo.DbMongo, "get_one") + def test_get_vdur(self, get_one): + get_one.return_value = {'_id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01', + '_admin': { + 'projects_read': ['admin'], 'created': 1526044312.102287, + 'modified': 1526044312.102287, 'projects_write': ['admin'] + }, + 'vim-account-id': 'c1740601-7287-48c8-a2c9-bce8fee459eb', + 'nsr-id-ref': '5ec3f571-d540-4cb0-9992-971d1b08312e', + 'vdur': [ + { + 'internal-connection-point': [], + 'vdu-id-ref': 'ubuntuvnf_vnfd-VM', + 'id': 'ffd73f33-c8bb-4541-a977-44dcc3cbe28d', + 'vim-id': '27042672-5190-4209-b844-95bbaeea7ea7', + 'name': 'ubuntuvnf_vnfd-VM' + } + ], + 'vnfd-ref': 'ubuntuvnf_vnfd', + 'member-vnf-index-ref': '1', + 'created-time': 1526044312.0999322, + 'vnfd-id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01', + 'id': 'a314c865-aee7-4d9b-9c9d-079d7f857f01'} + common_db_client = CommonDbClient(self.config) + vdur = common_db_client.get_vdur('5ec3f571-d540-4cb0-9992-971d1b08312e', '1', 'ubuntuvnf_vnfd-VM') + expected_vdur = { + 'internal-connection-point': [], + 'vdu-id-ref': 'ubuntuvnf_vnfd-VM', + 'id': 'ffd73f33-c8bb-4541-a977-44dcc3cbe28d', + 'vim-id': '27042672-5190-4209-b844-95bbaeea7ea7', + 'name': 'ubuntuvnf_vnfd-VM' + } + + self.assertDictEqual(vdur, expected_vdur) diff --git a/osm_mon/tests/unit/core/test_message_bus_client.py b/osm_mon/tests/unit/core/test_message_bus_client.py new file mode 100644 index 0000000..292fbe3 --- /dev/null +++ b/osm_mon/tests/unit/core/test_message_bus_client.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import asyncio +from unittest import TestCase, mock + +from osm_common.msgkafka import MsgKafka + +from osm_mon.core.message_bus_client import MessageBusClient +from osm_mon.core.config import Config + + +class TestMessageBusClient(TestCase): + + def setUp(self): + self.config = Config() + self.config.set('message', 'driver', 'kafka') + self.loop = asyncio.new_event_loop() + + @mock.patch.object(MsgKafka, 'aioread') + def test_aioread(self, aioread): + async def mock_callback(): + pass + + future = asyncio.Future(loop=self.loop) + future.set_result('mock') + aioread.return_value = future + msg_bus = MessageBusClient(self.config, loop=self.loop) + topic = 'test_topic' + self.loop.run_until_complete(msg_bus.aioread([topic], mock_callback)) + aioread.assert_called_with(['test_topic'], self.loop, aiocallback=mock_callback) + + @mock.patch.object(MsgKafka, 'aiowrite') + def test_aiowrite(self, aiowrite): + future = asyncio.Future(loop=self.loop) + future.set_result('mock') + aiowrite.return_value = future + msg_bus = MessageBusClient(self.config, loop=self.loop) + topic = 'test_topic' + key = 'test_key' + msg = {'test': 'test_msg'} + self.loop.run_until_complete(msg_bus.aiowrite(topic, key, msg)) + aiowrite.assert_called_with(topic, key, msg, self.loop) + + @mock.patch.object(MsgKafka, 'aioread') + def test_aioread_once(self, aioread): + future = asyncio.Future(loop=self.loop) + future.set_result('mock') + aioread.return_value = future + msg_bus = MessageBusClient(self.config, loop=self.loop) + topic = 'test_topic' + self.loop.run_until_complete(msg_bus.aioread_once(topic)) + aioread.assert_called_with('test_topic', self.loop) diff --git a/osm_mon/tests/unit/evaluator/__init__.py b/osm_mon/tests/unit/evaluator/__init__.py new file mode 100644 index 0000000..ce869a3 --- /dev/null +++ b/osm_mon/tests/unit/evaluator/__init__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com diff --git a/osm_mon/tests/unit/evaluator/test_evaluator.py b/osm_mon/tests/unit/evaluator/test_evaluator.py new file mode 100644 index 0000000..b20f602 --- /dev/null +++ b/osm_mon/tests/unit/evaluator/test_evaluator.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +from unittest import TestCase, mock + +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +from osm_mon.core.message_bus_client import MessageBusClient +from osm_mon.evaluator.evaluator import AlarmStatus, Evaluator +from osm_mon.evaluator.service import EvaluatorService + + +@mock.patch.object(CommonDbClient, "__init__", lambda *args, **kwargs: None) +@mock.patch.object(MessageBusClient, "__init__", lambda *args, **kwargs: None) +class EvaluatorTest(TestCase): + def setUp(self): + super().setUp() + self.config = Config() + + @mock.patch('multiprocessing.Process') + @mock.patch.object(Evaluator, "notify_alarm") + @mock.patch.object(EvaluatorService, "evaluate_alarms") + @mock.patch('osm_mon.core.database.db') + def test_evaluate(self, db, evaluate_alarms, notify_alarm, process): + mock_alarm = mock.Mock() + mock_alarm.operation = 'gt' + mock_alarm.threshold = 50.0 + evaluate_alarms.return_value = [(mock_alarm, AlarmStatus.ALARM)] + + evaluator = Evaluator(self.config) + evaluator.evaluate() + + process.assert_called_with(target=notify_alarm, args=(mock_alarm, AlarmStatus.ALARM)) diff --git a/osm_mon/tests/unit/evaluator/test_evaluator_service.py b/osm_mon/tests/unit/evaluator/test_evaluator_service.py new file mode 100644 index 0000000..88e3b4d --- /dev/null +++ b/osm_mon/tests/unit/evaluator/test_evaluator_service.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +from unittest import TestCase, mock + +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +from osm_mon.core.database import AlarmRepository +from osm_mon.core.message_bus_client import MessageBusClient +from osm_mon.evaluator.evaluator import AlarmStatus +from osm_mon.evaluator.service import EvaluatorService + +vnfr_record_mock = { + "_id": "0d9d06ad-3fc2-418c-9934-465e815fafe2", + "ip-address": "192.168.160.2", + "created-time": 1535392482.0044956, + "vim-account-id": "be48ae31-1d46-4892-a4b4-d69abd55714b", + "vdur": [ + { + "interfaces": [ + { + "mac-address": "fa:16:3e:71:fd:b8", + "name": "eth0", + "ip-address": "192.168.160.2" + } + ], + "status": "ACTIVE", + "vim-id": "63a65636-9fc8-4022-b070-980823e6266a", + "name": "cirros_ns-1-cirros_vnfd-VM-1", + "status-detailed": None, + "ip-address": "192.168.160.2", + "vdu-id-ref": "cirros_vnfd-VM" + } + ], + "id": "0d9d06ad-3fc2-418c-9934-465e815fafe2", + "vnfd-ref": "cirros_vdu_scaling_vnf", + "vnfd-id": "63f44c41-45ee-456b-b10d-5f08fb1796e0", + "_admin": { + "created": 1535392482.0067868, + "projects_read": [ + "admin" + ], + "modified": 1535392482.0067868, + "projects_write": [ + "admin" + ] + }, + "nsr-id-ref": "87776f33-b67c-417a-8119-cb08e4098951", + "member-vnf-index-ref": "1", + "connection-point": [ + { + "name": "eth0", + "id": None, + "connection-point-id": None + } + ] +} + +vnfd_record_mock = { + "_id": "63f44c41-45ee-456b-b10d-5f08fb1796e0", + "name": "cirros_vdu_scaling_vnf", + "vendor": "OSM", + "vdu": [ + { + "name": "cirros_vnfd-VM", + "monitoring-param": [ + { + "id": "cirros_vnfd-VM_memory_util", + "nfvi-metric": "average_memory_utilization" + } + ], + "vm-flavor": { + "vcpu-count": 1, + "memory-mb": 256, + "storage-gb": 2 + }, + "description": "cirros_vnfd-VM", + "count": 1, + "id": "cirros_vnfd-VM", + "interface": [ + { + "name": "eth0", + "external-connection-point-ref": "eth0", + "type": "EXTERNAL", + "virtual-interface": { + "bandwidth": "0", + "type": "VIRTIO", + "vpci": "0000:00:0a.0" + } + } + ], + "image": "cirros034" + } + ], + "monitoring-param": [ + { + "id": "cirros_vnf_memory_util", + "name": "cirros_vnf_memory_util", + "aggregation-type": "AVERAGE", + "vdu-monitoring-param": { + "vdu-monitoring-param-ref": "cirros_vnfd-VM_memory_util", + "vdu-ref": "cirros_vnfd-VM" + } + } + ], + "description": "Simple VNF example with a cirros and a scaling group descriptor", + "id": "cirros_vdu_scaling_vnf", + "logo": "cirros-64.png", + "version": "1.0", + "connection-point": [ + { + "name": "eth0", + "type": "VPORT" + } + ], + "mgmt-interface": { + "cp": "eth0" + }, + "short-name": "cirros_vdu_scaling_vnf", + "_admin": {} +} + + +@mock.patch.object(CommonDbClient, "__init__", lambda *args, **kwargs: None) +@mock.patch.object(MessageBusClient, "__init__", lambda *args, **kwargs: None) +class EvaluatorTest(TestCase): + def setUp(self): + super().setUp() + self.config = Config() + + @mock.patch.object(EvaluatorService, "_get_metric_value") + @mock.patch('osm_mon.core.database.db') + def test_evaluate_metric(self, db, get_metric_value): + mock_alarm = mock.Mock() + mock_alarm.operation = 'gt' + mock_alarm.threshold = 50.0 + get_metric_value.return_value = 100.0 + + service = EvaluatorService(self.config) + service.queue = mock.Mock() + service._evaluate_metric('test_id', 1, 'test_name', 'test_metric_name', mock_alarm) + service.queue.put.assert_called_with((mock_alarm, AlarmStatus.ALARM)) + service.queue.reset_mock() + + mock_alarm.operation = 'lt' + service._evaluate_metric('test_id', 1, 'test_name', 'test_metric_name', mock_alarm) + service.queue.put.assert_called_with((mock_alarm, AlarmStatus.OK)) + service.queue.reset_mock() + + get_metric_value.return_value = None + service._evaluate_metric('test_id', 1, 'test_name', 'test_metric_name', mock_alarm) + service.queue.put.assert_called_with((mock_alarm, AlarmStatus.INSUFFICIENT)) + + @mock.patch('multiprocessing.Process') + @mock.patch.object(EvaluatorService, "_evaluate_metric") + @mock.patch.object(CommonDbClient, "get_vnfd") + @mock.patch.object(CommonDbClient, "get_vnfr") + @mock.patch.object(AlarmRepository, "list") + @mock.patch('osm_mon.core.database.db') + def test_evaluate(self, db, alarm_list, get_vnfr, get_vnfd, evaluate_metric, proccess): + mock_alarm = mock.Mock() + mock_alarm.vdur_name = 'cirros_ns-1-cirros_vnfd-VM-1' + mock_alarm.monitoring_param = 'cirros_vnf_memory_util' + alarm_list.return_value = [mock_alarm] + get_vnfr.return_value = vnfr_record_mock + get_vnfd.return_value = vnfd_record_mock + + evaluator = EvaluatorService(self.config) + evaluator.evaluate_alarms() + + proccess.assert_called_with(target=evaluate_metric, args=( + '87776f33-b67c-417a-8119-cb08e4098951', '1', 'cirros_ns-1-cirros_vnfd-VM-1', 'average_memory_utilization', + mock_alarm)) diff --git a/osm_mon/tests/unit/server/__init__.py b/osm_mon/tests/unit/server/__init__.py new file mode 100644 index 0000000..ce869a3 --- /dev/null +++ b/osm_mon/tests/unit/server/__init__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com diff --git a/osm_mon/tests/unit/server/test_server_service.py b/osm_mon/tests/unit/server/test_server_service.py new file mode 100644 index 0000000..9c4dcd1 --- /dev/null +++ b/osm_mon/tests/unit/server/test_server_service.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +from unittest import TestCase, mock + +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +from osm_mon.core.database import VimCredentialsRepository, VimCredentials +from osm_mon.server.service import ServerService + + +@mock.patch.object(CommonDbClient, "__init__", lambda *args, **kwargs: None) +class ServerServiceTest(TestCase): + def setUp(self): + super().setUp() + self.config = Config() + + @mock.patch.object(CommonDbClient, "decrypt_vim_password") + @mock.patch.object(VimCredentialsRepository, "upsert") + @mock.patch('osm_mon.core.database.db') + def test_upsert_vim_account(self, db, upsert_credentials, decrypt_vim_password): + def _mock_decrypt_vim_password(password: str, schema_version: str, vim_uuid: str): + return password.replace('encrypted', 'decrypted') + + decrypt_vim_password.side_effect = _mock_decrypt_vim_password + + mock_config = { + 'admin_password': 'encrypted_admin_password', + 'nsx_password': 'encrypted_nsx_password', + 'vcenter_password': 'encrypted_vcenter_password' + } + + mock_expected_config = { + 'admin_password': 'decrypted_admin_password', + 'nsx_password': 'decrypted_nsx_password', + 'vcenter_password': 'decrypted_vcenter_password' + } + + service = ServerService(self.config) + service.upsert_vim_account('test_uuid', 'test_name', 'test_type', 'test_url', 'test_user', 'encrypted_password', + 'test_tenant_name', '1.1', mock_config) + + upsert_credentials.assert_called_with( + uuid=mock.ANY, + name='test_name', + type='test_type', + url='test_url', + user='test_user', + password='decrypted_password', + tenant_name='test_tenant_name', + config=json.dumps(mock_expected_config) + ) + + @mock.patch.object(VimCredentialsRepository, "get") + @mock.patch('osm_mon.core.database.db') + def test_delete_vim_account(self, db, get_credentials): + mock_creds = mock.Mock() + get_credentials.return_value = mock_creds + + service = ServerService(self.config) + service.delete_vim_account('test_uuid') + + get_credentials.assert_called_with(VimCredentials.uuid == 'test_uuid') + mock_creds.delete_instance.assert_called_with() diff --git a/setup.cfg b/setup.cfg index df6b252..61ab554 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ classifier = Programming Language :: Python :: 3.5 [test] -test_suite=test +test_suite=osm_mon.tests [files] packages =