From: lavado Date: Fri, 29 Nov 2019 13:00:24 +0000 (+0100) Subject: Revert "Migrates alarms to MongoDB" X-Git-Tag: v7.0.0rc1~15^2 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=refs%2Fchanges%2F62%2F8262%2F2;p=osm%2FMON.git Revert "Migrates alarms to MongoDB" This reverts commit 0085a4a8870a73a9186b0a2305ba00bae44a6ea0. Change-Id: Ifd94058969c1480863a67529a9d5ed70e5e062c3 Signed-off-by: Gianpietro Lavado --- diff --git a/docs/architecture.md b/docs/architecture.md index cec6697..8e90ab9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -50,7 +50,7 @@ MON Server subscribes to the message bus and waits for the following messages: It performs the corresponding action and sends a response through a unique topic made of 'alarm_response_' plus a correlation_id, which is a field contained in the request message. -Alarms are stored in MON database in the OSM MongoDB engine. +Alarms are stored in MON database in the OSM MySQL engine. ## MON Collector diff --git a/docs/assets/MON_Evaluator_Evaluate_Alarm.jpg b/docs/assets/MON_Evaluator_Evaluate_Alarm.jpg index 0e2e676..930380f 100644 Binary files a/docs/assets/MON_Evaluator_Evaluate_Alarm.jpg and b/docs/assets/MON_Evaluator_Evaluate_Alarm.jpg differ diff --git a/docs/assets/MON_Overview_Diagram.jpg b/docs/assets/MON_Overview_Diagram.jpg index e6f8b78..c804347 100644 Binary files a/docs/assets/MON_Overview_Diagram.jpg and b/docs/assets/MON_Overview_Diagram.jpg differ diff --git a/docs/assets/MON_Server_Create_Alarm.jpg b/docs/assets/MON_Server_Create_Alarm.jpg index 5feeebc..a4b6b9d 100644 Binary files a/docs/assets/MON_Server_Create_Alarm.jpg and b/docs/assets/MON_Server_Create_Alarm.jpg differ diff --git a/osm_mon/cmd/mon_collector.py b/osm_mon/cmd/mon_collector.py index 94c3883..3e493de 100644 --- a/osm_mon/cmd/mon_collector.py +++ b/osm_mon/cmd/mon_collector.py @@ -27,11 +27,12 @@ import sys from osm_mon.collector.collector import Collector from osm_mon.core.config import Config +from osm_mon.core.database import DatabaseManager def main(): - parser = argparse.ArgumentParser(prog='osm-mon-collector') - parser.add_argument('--config-file', nargs='?', help='MON configuration file') + parser = argparse.ArgumentParser(prog='osm-policy-agent') + parser.add_argument('--config-file', nargs='?', help='POL configuration file') args = parser.parse_args() cfg = Config(args.config_file) @@ -47,6 +48,9 @@ def main(): log.info("Starting MON Collector...") log.debug("Config: %s", cfg.conf) log.info("Initializing database...") + db_manager = DatabaseManager(cfg) + db_manager.create_tables() + log.info("Database initialized correctly.") collector = Collector(cfg) collector.collect_forever() diff --git a/osm_mon/cmd/mon_evaluator.py b/osm_mon/cmd/mon_evaluator.py index ba9a420..3835d7e 100644 --- a/osm_mon/cmd/mon_evaluator.py +++ b/osm_mon/cmd/mon_evaluator.py @@ -26,12 +26,13 @@ import logging import sys from osm_mon.core.config import Config +from osm_mon.core.database import DatabaseManager from osm_mon.evaluator.evaluator import Evaluator def main(): - parser = argparse.ArgumentParser(prog='osm-mon-evaluator') - parser.add_argument('--config-file', nargs='?', help='MON configuration file') + parser = argparse.ArgumentParser(prog='osm-policy-agent') + parser.add_argument('--config-file', nargs='?', help='POL configuration file') args = parser.parse_args() cfg = Config(args.config_file) @@ -47,6 +48,9 @@ def main(): log.info("Starting MON Evaluator...") log.debug("Config: %s", cfg.conf) log.info("Initializing database...") + db_manager = DatabaseManager(cfg) + db_manager.create_tables() + log.info("Database initialized correctly.") evaluator = Evaluator(cfg) evaluator.evaluate_forever() diff --git a/osm_mon/cmd/mon_server.py b/osm_mon/cmd/mon_server.py index e05f4b8..6698339 100644 --- a/osm_mon/cmd/mon_server.py +++ b/osm_mon/cmd/mon_server.py @@ -27,12 +27,13 @@ import logging import sys from osm_mon.core.config import Config +from osm_mon.core.database import DatabaseManager from osm_mon.server.server import Server def main(): - parser = argparse.ArgumentParser(prog='osm-mon-server') - parser.add_argument('--config-file', nargs='?', help='MON configuration file') + parser = argparse.ArgumentParser(prog='osm-policy-agent') + parser.add_argument('--config-file', nargs='?', help='POL configuration file') args = parser.parse_args() cfg = Config(args.config_file) @@ -48,6 +49,9 @@ def main(): log.info("Starting MON Server...") log.debug("Config: %s", cfg.conf) log.info("Initializing database...") + db_manager = DatabaseManager(cfg) + db_manager.create_tables() + log.info("Database initialized correctly.") loop = asyncio.get_event_loop() server = Server(cfg, loop) server.run() diff --git a/osm_mon/core/common_db.py b/osm_mon/core/common_db.py index 5fa27eb..8df8672 100644 --- a/osm_mon/core/common_db.py +++ b/osm_mon/core/common_db.py @@ -21,12 +21,9 @@ # For those usages not covered by the Apache License, Version 2.0 please # contact: bdiaz@whitestack.com or glavado@whitestack.com ## -from typing import List - from osm_common import dbmongo, dbmemory from osm_mon.core.config import Config -from osm_mon.core.models import Alarm class CommonDbClient: @@ -118,16 +115,3 @@ class CommonDbClient: def get_sdnc(self, sdnc_id: str): return self.common_db.get_one('sdns', {'_id': sdnc_id}) - - def create_alarm(self, alarm: Alarm): - return self.common_db.create('alarms', alarm.to_dict()) - - def delete_alarm(self, alarm_uuid: str): - return self.common_db.del_one('alarms', {'uuid': alarm_uuid}) - - def get_alarms(self) -> List[Alarm]: - alarms = [] - alarm_dicts = self.common_db.get_list('alarms') - for alarm_dict in alarm_dicts: - alarms.append(Alarm.from_dict(alarm_dict)) - return alarms diff --git a/osm_mon/core/database.py b/osm_mon/core/database.py new file mode 100644 index 0000000..61bd180 --- /dev/null +++ b/osm_mon/core/database.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## + +import logging +import os +from typing import Iterable + +from peewee import CharField, FloatField, Model, AutoField, Proxy, ForeignKeyField +from peewee_migrate import Router +from playhouse.db_url import connect + +from osm_mon import migrations +from osm_mon.core.config import Config + +log = logging.getLogger(__name__) + +db = Proxy() + + +class BaseModel(Model): + id = AutoField(primary_key=True) + + class Meta: + database = db + + +class Alarm(BaseModel): + uuid = CharField(unique=True) + name = CharField() + severity = CharField() + threshold = FloatField() + operation = CharField() + statistic = CharField() + metric = CharField() + + +class AlarmTag(BaseModel): + name = CharField() + value = CharField() + alarm = ForeignKeyField(Alarm, related_name='tags', on_delete='CASCADE') + + +class DatabaseManager: + def __init__(self, config: Config): + db.initialize(connect(config.get('sql', 'database_uri'))) + + def create_tables(self) -> None: + db.connect() + with db.atomic(): + router = Router(db, os.path.dirname(migrations.__file__)) + router.run() + db.close() + + +class AlarmTagRepository: + @staticmethod + def create(**query) -> Alarm: + return AlarmTag.create(**query) + + +class AlarmRepository: + @staticmethod + def create(**query) -> Alarm: + return Alarm.create(**query) + + @staticmethod + def get(*expressions) -> Alarm: + return Alarm.select().where(*expressions).get() + + @staticmethod + def list(*expressions) -> Iterable[Alarm]: + if expressions == (): + return Alarm.select() + else: + return Alarm.select().where(*expressions) diff --git a/osm_mon/core/models.py b/osm_mon/core/models.py deleted file mode 100644 index 1810c9b..0000000 --- a/osm_mon/core/models.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# For those usages not covered by the Apache License, Version 2.0 please -# contact: bdiaz@whitestack.com or glavado@whitestack.com -## -import uuid - - -class Alarm: - - def __init__(self, name: str = None, severity: str = None, threshold: float = None, operation: str = None, - statistic: str = None, metric: str = None, tags: dict = {}): - self.uuid = str(uuid.uuid4()) - self.name = name - self.severity = severity - self.threshold = threshold - self.operation = operation - self.statistic = statistic - self.metric = metric - self.tags = tags - - def to_dict(self) -> dict: - alarm = { - 'uuid': self.uuid, - 'name': self.name, - 'severity': self.severity, - 'threshold': self.threshold, - 'statistic': self.statistic, - 'metric': self.metric, - 'tags': self.tags - } - return alarm - - @staticmethod - def from_dict(data: dict): - alarm = Alarm() - alarm.uuid = data.get('uuid', str(uuid.uuid4())) - alarm.name = data.get('name') - alarm.severity = data.get('severity') - alarm.threshold = data.get('threshold') - alarm.statistic = data.get('statistic') - alarm.metric = data.get('metric') - alarm.tags = data.get('tags') - return alarm diff --git a/osm_mon/evaluator/evaluator.py b/osm_mon/evaluator/evaluator.py index cc9a8ad..2f22625 100644 --- a/osm_mon/evaluator/evaluator.py +++ b/osm_mon/evaluator/evaluator.py @@ -28,8 +28,8 @@ import time import peewee from osm_mon.core.config import Config +from osm_mon.core.database import Alarm from osm_mon.core.message_bus_client import MessageBusClient -from osm_mon.core.models import Alarm from osm_mon.core.response import ResponseBuilder from osm_mon.evaluator.service import EvaluatorService, AlarmStatus diff --git a/osm_mon/evaluator/service.py b/osm_mon/evaluator/service.py index b3b0d26..de3798b 100644 --- a/osm_mon/evaluator/service.py +++ b/osm_mon/evaluator/service.py @@ -25,9 +25,10 @@ import multiprocessing from enum import Enum from typing import Tuple, List +from osm_mon.core import database from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config -from osm_mon.core.models import Alarm +from osm_mon.core.database import Alarm, AlarmRepository from osm_mon.evaluator.backends.prometheus import PrometheusBackend log = logging.getLogger(__name__) @@ -56,11 +57,11 @@ class EvaluatorService: return BACKENDS[self.conf.get('evaluator', 'backend')](self.conf).get_metric_value(metric_name, tags) def _evaluate_metric(self, - alarm: Alarm): + alarm: Alarm, tags: dict): log.debug("_evaluate_metric") - metric_value = self._get_metric_value(alarm.metric, alarm.tags) + metric_value = self._get_metric_value(alarm.metric, tags) if metric_value is None: - log.warning("No metric result for alarm %s", alarm.uuid) + log.warning("No metric result for alarm %s", alarm.id) self.queue.put((alarm, AlarmStatus.INSUFFICIENT)) else: if alarm.operation.upper() == 'GT': @@ -77,15 +78,25 @@ class EvaluatorService: def evaluate_alarms(self) -> List[Tuple[Alarm, AlarmStatus]]: log.debug('evaluate_alarms') processes = [] - for alarm in self.common_db.get_alarms(): - p = multiprocessing.Process(target=self._evaluate_metric, - args=(alarm,)) - processes.append(p) - p.start() - - for process in processes: - process.join(timeout=10) - alarms_tuples = [] - while not self.queue.empty(): - alarms_tuples.append(self.queue.get()) - return alarms_tuples + database.db.connect() + try: + with database.db.atomic(): + for alarm in AlarmRepository.list(): + # Tags need to be passed inside a dict to avoid database locking issues related to process forking + tags = {} + for tag in alarm.tags: + tags[tag.name] = tag.value + p = multiprocessing.Process(target=self._evaluate_metric, + args=(alarm, tags)) + processes.append(p) + p.start() + + for process in processes: + process.join(timeout=10) + alarms_tuples = [] + log.info("Appending alarms to queue") + while not self.queue.empty(): + alarms_tuples.append(self.queue.get()) + return alarms_tuples + finally: + database.db.close() diff --git a/osm_mon/migrations/001_initial.py b/osm_mon/migrations/001_initial.py new file mode 100644 index 0000000..346e9c1 --- /dev/null +++ b/osm_mon/migrations/001_initial.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +"""Peewee migrations -- 001_initial.py. + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" + +import peewee as pw + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + """Write your migrations here.""" + + @migrator.create_model + class Alarm(pw.Model): + id = pw.AutoField() + uuid = pw.CharField(max_length=255, unique=True) + name = pw.CharField(max_length=255) + severity = pw.CharField(max_length=255) + threshold = pw.FloatField() + operation = pw.CharField(max_length=255) + statistic = pw.CharField(max_length=255) + monitoring_param = pw.CharField(max_length=255) + vdur_name = pw.CharField(max_length=255) + vnf_member_index = pw.CharField(max_length=255) + nsr_id = pw.CharField(max_length=255) + + class Meta: + table_name = "alarm" + + @migrator.create_model + class BaseModel(pw.Model): + id = pw.AutoField() + + class Meta: + table_name = "basemodel" + + @migrator.create_model + class VimCredentials(pw.Model): + id = pw.AutoField() + uuid = pw.CharField(max_length=255, unique=True) + name = pw.CharField(max_length=255) + type = pw.CharField(max_length=255) + url = pw.CharField(max_length=255) + user = pw.CharField(max_length=255) + password = pw.CharField(max_length=255) + tenant_name = pw.CharField(max_length=255) + config = pw.TextField() + + class Meta: + table_name = "vimcredentials" + + +def rollback(migrator, database, fake=False, **kwargs): + """Write your rollback migrations here.""" + + migrator.remove_model('vimcredentials') + + migrator.remove_model('basemodel') + + migrator.remove_model('alarm') diff --git a/osm_mon/migrations/002_add_alarm_tags.py b/osm_mon/migrations/002_add_alarm_tags.py new file mode 100644 index 0000000..22f5de8 --- /dev/null +++ b/osm_mon/migrations/002_add_alarm_tags.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +"""Peewee migrations -- 002_add_alarm_tags.py. + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" + +import peewee as pw + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + """Write your migrations here.""" + + @migrator.create_model + class AlarmTag(pw.Model): + id = pw.AutoField() + name = pw.CharField(max_length=255) + value = pw.CharField(max_length=255) + alarm = pw.ForeignKeyField(backref='tags', column_name='alarm_id', field='id', + model=migrator.orm['alarm'], on_delete='CASCADE') + + class Meta: + table_name = "alarmtag" + + +def rollback(migrator, database, fake=False, **kwargs): + """Write your rollback migrations here.""" + + migrator.remove_model('alarmtag') diff --git a/osm_mon/migrations/003_rename_monitoring_param.py b/osm_mon/migrations/003_rename_monitoring_param.py new file mode 100644 index 0000000..2d5108f --- /dev/null +++ b/osm_mon/migrations/003_rename_monitoring_param.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +"""Peewee migrations -- 003_rename_monitoring_param.py. + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" + +import peewee as pw + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + """Write your migrations here.""" + + migrator.rename_field('alarm', 'monitoring_param', 'metric') + + +def rollback(migrator, database, fake=False, **kwargs): + """Write your rollback migrations here.""" + + migrator.rename_field('alarm', 'metric', 'monitoring_param') diff --git a/osm_mon/migrations/004_remove_alarm_fields.py b/osm_mon/migrations/004_remove_alarm_fields.py new file mode 100644 index 0000000..b9477ac --- /dev/null +++ b/osm_mon/migrations/004_remove_alarm_fields.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +"""Peewee migrations -- 004_remove_alarm_fields.py. + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" + +import peewee as pw + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + """Write your migrations here.""" + + migrator.remove_fields('alarm', 'vdur_name', 'vnf_member_index', 'nsr_id') + + +def rollback(migrator, database, fake=False, **kwargs): + """Write your rollback migrations here.""" + + migrator.add_fields('alarm', + vdur_name=pw.CharField(max_length=255), + vnf_member_index=pw.CharField(max_length=255), + nsr_id=pw.CharField(max_length=255)) diff --git a/osm_mon/migrations/__init__.py b/osm_mon/migrations/__init__.py new file mode 100644 index 0000000..d81308a --- /dev/null +++ b/osm_mon/migrations/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## diff --git a/osm_mon/migrations/conf.py b/osm_mon/migrations/conf.py new file mode 100644 index 0000000..c6bda5a --- /dev/null +++ b/osm_mon/migrations/conf.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import os + +DATABASE = os.getenv('OSMMON_SQL_DATABASE_URI', 'sqlite://') diff --git a/osm_mon/server/service.py b/osm_mon/server/service.py index 60cb3ec..1d546e3 100755 --- a/osm_mon/server/service.py +++ b/osm_mon/server/service.py @@ -21,10 +21,12 @@ # contact: bdiaz@whitestack.com or glavado@whitestack.com ## import logging +import uuid +from osm_mon.core import database from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config -from osm_mon.core.models import Alarm +from osm_mon.core.database import AlarmRepository, Alarm, AlarmTagRepository log = logging.getLogger(__name__) @@ -36,16 +38,41 @@ class ServerService: def create_alarm(self, name: str, - threshold: float, + threshold: str, operation: str, severity: str, statistic: str, metric_name: str, tags: dict) -> Alarm: - alarm = Alarm(name, severity, threshold, operation, statistic, metric_name, tags) - self.common_db.create_alarm(alarm) - return alarm + database.db.connect() + try: + with database.db.atomic(): + alarm = AlarmRepository.create( + uuid=str(uuid.uuid4()), + name=name, + threshold=threshold, + operation=operation.lower(), + severity=severity.lower(), + statistic=statistic.lower(), + metric=metric_name + ) + for k, v in tags.items(): + AlarmTagRepository.create( + name=k, + value=v, + alarm=alarm + ) + return alarm + + finally: + database.db.close() def delete_alarm(self, alarm_uuid: str) -> None: - self.common_db.delete_alarm(alarm_uuid) + database.db.connect() + try: + with database.db.atomic(): + alarm = AlarmRepository.get(Alarm.uuid == alarm_uuid) + alarm.delete_instance() + finally: + database.db.close() diff --git a/osm_mon/tests/unit/core/test_common_db_client.py b/osm_mon/tests/unit/core/test_common_db_client.py index 7102226..e584149 100644 --- a/osm_mon/tests/unit/core/test_common_db_client.py +++ b/osm_mon/tests/unit/core/test_common_db_client.py @@ -27,7 +27,6 @@ from osm_common import dbmongo from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config -from osm_mon.core.models import Alarm class CommonDbClientTest(unittest.TestCase): @@ -147,37 +146,3 @@ class CommonDbClientTest(unittest.TestCase): decrypt_vim_password.assert_any_call('vim_password', schema_version, vim_id) self.assertRaises(AssertionError, decrypt_vim_password.assert_any_call, 'vrops_password', schema_version, vim_id) - - @mock.patch.object(dbmongo.DbMongo, "db_connect", mock.Mock()) - @mock.patch.object(dbmongo.DbMongo, "get_list") - def test_get_alarms(self, get_list): - get_list.return_value = [{ - 'uuid': '1', - 'name': 'name', - 'severity': 'severity', - 'threshold': 50, - 'operation': 'operation', - 'statistic': 'statistic', - 'tags': {}, - }] - - common_db_client = CommonDbClient(self.config) - alarms = common_db_client.get_alarms() - self.assertEqual('1', alarms[0].uuid) - - @mock.patch.object(dbmongo.DbMongo, "db_connect", mock.Mock()) - @mock.patch.object(dbmongo.DbMongo, "create") - def test_create_alarm(self, create): - alarm = Alarm('name', 'severity', 50.0, 'operation', 'statistic', 'metric', {}) - alarm.uuid = '1' - common_db_client = CommonDbClient(self.config) - common_db_client.create_alarm(alarm) - create.assert_called_with('alarms', {'tags': {}, 'threshold': 50.0, 'metric': 'metric', 'severity': 'severity', - 'statistic': 'statistic', 'name': 'name', 'uuid': '1'}) - - @mock.patch.object(dbmongo.DbMongo, "db_connect", mock.Mock()) - @mock.patch.object(dbmongo.DbMongo, "del_one") - def test_delete_alarm(self, delete): - common_db_client = CommonDbClient(self.config) - common_db_client.delete_alarm('1') - delete.assert_called_with('alarms', {'uuid': '1'}) diff --git a/osm_mon/tests/unit/evaluator/test_evaluator.py b/osm_mon/tests/unit/evaluator/test_evaluator.py index 43cec96..b20f602 100644 --- a/osm_mon/tests/unit/evaluator/test_evaluator.py +++ b/osm_mon/tests/unit/evaluator/test_evaluator.py @@ -39,7 +39,8 @@ class EvaluatorTest(TestCase): @mock.patch('multiprocessing.Process') @mock.patch.object(Evaluator, "notify_alarm") @mock.patch.object(EvaluatorService, "evaluate_alarms") - def test_evaluate(self, evaluate_alarms, notify_alarm, process): + @mock.patch('osm_mon.core.database.db') + def test_evaluate(self, db, evaluate_alarms, notify_alarm, process): mock_alarm = mock.Mock() mock_alarm.operation = 'gt' mock_alarm.threshold = 50.0 diff --git a/osm_mon/tests/unit/evaluator/test_evaluator_service.py b/osm_mon/tests/unit/evaluator/test_evaluator_service.py index bc93046..e09418d 100644 --- a/osm_mon/tests/unit/evaluator/test_evaluator_service.py +++ b/osm_mon/tests/unit/evaluator/test_evaluator_service.py @@ -24,6 +24,7 @@ from unittest import TestCase, mock from osm_mon.core.common_db import CommonDbClient from osm_mon.core.config import Config +from osm_mon.core.database import AlarmRepository, AlarmTag from osm_mon.core.message_bus_client import MessageBusClient from osm_mon.evaluator.backends.prometheus import PrometheusBackend from osm_mon.evaluator.evaluator import AlarmStatus @@ -148,7 +149,8 @@ class EvaluatorTest(TestCase): self.config = Config() @mock.patch.object(EvaluatorService, "_get_metric_value") - def test_evaluate_metric(self, get_metric_value): + @mock.patch('osm_mon.core.database.db') + def test_evaluate_metric(self, db, get_metric_value): mock_alarm = mock.Mock() mock_alarm.operation = 'gt' mock_alarm.threshold = 50.0 @@ -157,29 +159,33 @@ class EvaluatorTest(TestCase): service = EvaluatorService(self.config) service.queue = mock.Mock() - service._evaluate_metric(mock_alarm) + service._evaluate_metric(mock_alarm, {}) service.queue.put.assert_called_with((mock_alarm, AlarmStatus.ALARM)) service.queue.reset_mock() mock_alarm.operation = 'lt' - service._evaluate_metric(mock_alarm) + service._evaluate_metric(mock_alarm, {}) service.queue.put.assert_called_with((mock_alarm, AlarmStatus.OK)) service.queue.reset_mock() get_metric_value.return_value = None - service._evaluate_metric(mock_alarm) + service._evaluate_metric(mock_alarm, {}) service.queue.put.assert_called_with((mock_alarm, AlarmStatus.INSUFFICIENT)) @mock.patch('multiprocessing.Process') @mock.patch.object(EvaluatorService, "_evaluate_metric") @mock.patch.object(CommonDbClient, "get_vnfd") @mock.patch.object(CommonDbClient, "get_vnfr") - @mock.patch.object(CommonDbClient, "get_alarms") - def test_evaluate_alarms(self, alarm_list, get_vnfr, get_vnfd, evaluate_metric, process): + @mock.patch.object(AlarmRepository, "list") + @mock.patch('osm_mon.core.database.db') + def test_evaluate(self, db, alarm_list, get_vnfr, get_vnfd, evaluate_metric, process): mock_alarm = mock.Mock() mock_alarm.vdur_name = 'cirros_ns-1-cirros_vnfd-VM-1' mock_alarm.monitoring_param = 'cirros_vnf_memory_util' - mock_alarm.tags = {'name': 'value'} + mock_tag = AlarmTag() + mock_tag.name = 'name' + mock_tag.value = 'value' + mock_alarm.tags = [mock_tag] alarm_list.return_value = [mock_alarm] get_vnfr.return_value = vnfr_record_mock get_vnfd.return_value = vnfd_record_mock @@ -187,10 +193,11 @@ class EvaluatorTest(TestCase): evaluator = EvaluatorService(self.config) evaluator.evaluate_alarms() - process.assert_called_with(target=evaluate_metric, args=(mock_alarm,)) + process.assert_called_with(target=evaluate_metric, args=(mock_alarm, {'name': 'value'})) @mock.patch.object(PrometheusBackend, "get_metric_value") - def test_get_metric_value_prometheus(self, get_metric_value): + @mock.patch('osm_mon.core.database.db') + def test_get_metric_value_prometheus(self, db, get_metric_value): self.config.set('evaluator', 'backend', 'prometheus') evaluator = EvaluatorService(self.config) evaluator._get_metric_value('test', {})