From 17ebe3794d0dc2712b1438f3fc5469ca10d752e1 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Tue, 25 Sep 2018 16:16:31 +0200 Subject: [PATCH] Revert "Removes POL code from MON repo" This reverts commit f315c3bf13c14d6a71c2909b4e4a8632f7df1c2e. --- policy_module/.gitignore | 60 +++++++ policy_module/Dockerfile | 39 +++++ policy_module/MANIFEST.in | 3 + policy_module/README.rst | 14 ++ policy_module/config.example | 4 + policy_module/osm_policy_module/__init__.py | 0 .../osm_policy_module/cmd/__init__.py | 0 .../cmd/policy_module_agent.py | 77 +++++++++ .../osm_policy_module/common/__init__.py | 0 .../osm_policy_module/common/alarm_config.py | 34 ++++ .../osm_policy_module/common/lcm_client.py | 57 +++++++ .../osm_policy_module/common/mon_client.py | 89 ++++++++++ .../osm_policy_module/core/__init__.py | 0 policy_module/osm_policy_module/core/agent.py | 157 ++++++++++++++++++ .../osm_policy_module/core/config.py | 73 ++++++++ .../osm_policy_module/core/database.py | 61 +++++++ .../osm_policy_module/core/singleton.py | 41 +++++ .../models/configure_scaling.json | 111 +++++++++++++ .../osm_policy_module/tests/__init__.py | 0 .../configure_scaling_full_example.json | 30 ++++ .../tests/integration/__init__.py | 0 .../test_scaling_config_kafka_msg.py | 66 ++++++++ .../osm_policy_module/tests/unit/__init__.py | 0 .../tests/unit/test_examples.py | 41 +++++ .../tests/unit/test_policy_agent.py | 44 +++++ policy_module/requirements.txt | 7 + policy_module/scripts/gen_config_from_env.sh | 19 +++ policy_module/setup.py | 72 ++++++++ 28 files changed, 1099 insertions(+) create mode 100644 policy_module/.gitignore create mode 100644 policy_module/Dockerfile create mode 100644 policy_module/MANIFEST.in create mode 100644 policy_module/README.rst create mode 100644 policy_module/config.example create mode 100644 policy_module/osm_policy_module/__init__.py create mode 100644 policy_module/osm_policy_module/cmd/__init__.py create mode 100644 policy_module/osm_policy_module/cmd/policy_module_agent.py create mode 100644 policy_module/osm_policy_module/common/__init__.py create mode 100644 policy_module/osm_policy_module/common/alarm_config.py create mode 100644 policy_module/osm_policy_module/common/lcm_client.py create mode 100644 policy_module/osm_policy_module/common/mon_client.py create mode 100644 policy_module/osm_policy_module/core/__init__.py create mode 100644 policy_module/osm_policy_module/core/agent.py create mode 100644 policy_module/osm_policy_module/core/config.py create mode 100644 policy_module/osm_policy_module/core/database.py create mode 100644 policy_module/osm_policy_module/core/singleton.py create mode 100644 policy_module/osm_policy_module/models/configure_scaling.json create mode 100644 policy_module/osm_policy_module/tests/__init__.py create mode 100644 policy_module/osm_policy_module/tests/examples/configure_scaling_full_example.json create mode 100644 policy_module/osm_policy_module/tests/integration/__init__.py create mode 100644 policy_module/osm_policy_module/tests/integration/test_scaling_config_kafka_msg.py create mode 100644 policy_module/osm_policy_module/tests/unit/__init__.py create mode 100644 policy_module/osm_policy_module/tests/unit/test_examples.py create mode 100644 policy_module/osm_policy_module/tests/unit/test_policy_agent.py create mode 100644 policy_module/requirements.txt create mode 100644 policy_module/scripts/gen_config_from_env.sh create mode 100644 policy_module/setup.py diff --git a/policy_module/.gitignore b/policy_module/.gitignore new file mode 100644 index 0000000..88a8391 --- /dev/null +++ b/policy_module/.gitignore @@ -0,0 +1,60 @@ +*.py[cod] + +# C extensions +*.so + +# log files +*.log + +# Packages +*.egg +*.egg-info +dist +build +.eggs +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +nohup.out + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.testrepository +.venv +.cache + +# Translations +*.mo + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.*sw? +.settings/ +__pycache__/ +.idea + +*.db +test.config \ No newline at end of file diff --git a/policy_module/Dockerfile b/policy_module/Dockerfile new file mode 100644 index 0000000..553ed7b --- /dev/null +++ b/policy_module/Dockerfile @@ -0,0 +1,39 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## + +FROM ubuntu:16.04 + +LABEL authors="Benjamín Díaz" + +RUN apt-get --yes update \ + && apt-get --yes install python3 python3-pip libmysqlclient-dev git \ + && pip3 install pip==9.0.3 + +COPY requirements.txt /policy_module/requirements.txt + +RUN pip3 install -r /policy_module/requirements.txt + +COPY . /policy_module + +RUN pip3 install /policy_module + +CMD bash /policy_module/scripts/gen_config_from_env.sh && osm-policy-agent --config osm_policy_agent.cfg diff --git a/policy_module/MANIFEST.in b/policy_module/MANIFEST.in new file mode 100644 index 0000000..06cf953 --- /dev/null +++ b/policy_module/MANIFEST.in @@ -0,0 +1,3 @@ +include requirements.txt +include README.rst +recursive-include osm_policy_module \ No newline at end of file diff --git a/policy_module/README.rst b/policy_module/README.rst new file mode 100644 index 0000000..5cb2fde --- /dev/null +++ b/policy_module/README.rst @@ -0,0 +1,14 @@ +Install +------------------------ + :: + + git clone https://osm.etsi.org/gerrit/osm/MON.git + cd MON/policy_module + pip install . + +Run +------------------------ + :: + + osm-policy-agent + diff --git a/policy_module/config.example b/policy_module/config.example new file mode 100644 index 0000000..45e4f17 --- /dev/null +++ b/policy_module/config.example @@ -0,0 +1,4 @@ +[policy_module] +kafka_server_host=localhost +kafka_server_port=9092 +log_dir= \ No newline at end of file diff --git a/policy_module/osm_policy_module/__init__.py b/policy_module/osm_policy_module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/policy_module/osm_policy_module/cmd/__init__.py b/policy_module/osm_policy_module/cmd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/policy_module/osm_policy_module/cmd/policy_module_agent.py b/policy_module/osm_policy_module/cmd/policy_module_agent.py new file mode 100644 index 0000000..ac03167 --- /dev/null +++ b/policy_module/osm_policy_module/cmd/policy_module_agent.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import argparse +import logging +import sys +import logstash + +from osm_policy_module.core.agent import PolicyModuleAgent +from osm_policy_module.core.config import Config +from osm_policy_module.core.database import DatabaseManager + + +def main(): + cfg = Config.instance() + parser = argparse.ArgumentParser(prog='pm-scaling-config-agent') + parser.add_argument('--config-file', nargs='?', help='Policy module agent configuration file') + args = parser.parse_args() + if args.config_file: + cfg.load_file(args.config_file) + # TODO: Handle different log levels in config + if cfg.get('policy_module', 'log_dir') == 'stdout': + logging.basicConfig(stream=sys.stdout, + format='%(asctime)s %(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p', + level=logging.INFO) + else: + logging.basicConfig(filename=cfg.get('policy_module', 'log_dir') + 'pm_agent.log', + format='%(asctime)s %(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p', filemode='a', + level=logging.INFO) + if cfg.get('policy_module', 'enable_logstash_handler') == 'true': + logstash_host = cfg.get('policy_module', 'logstash_host') + logstash_port = int(cfg.get('policy_module', 'logstash_port')) + root_logger = logging.getLogger() + root_logger.addHandler(logstash.TCPLogstashHandler(logstash_host, logstash_port, version=1)) + root_logger.info("Logstash handler configured.") + kafka_logger = logging.getLogger('kafka') + kafka_logger.setLevel(logging.WARN) + kafka_formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + kafka_handler = logging.StreamHandler(sys.stdout) + kafka_handler.setFormatter(kafka_formatter) + kafka_logger.addHandler(kafka_handler) + log = logging.getLogger(__name__) + log.info("Config: %s", cfg) + log.info("Syncing database...") + db_manager = DatabaseManager() + db_manager.create_tables() + log.info("Database synced correctly.") + log.info("Starting policy module agent...") + agent = PolicyModuleAgent() + agent.run() + + +if __name__ == '__main__': + main() diff --git a/policy_module/osm_policy_module/common/__init__.py b/policy_module/osm_policy_module/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/policy_module/osm_policy_module/common/alarm_config.py b/policy_module/osm_policy_module/common/alarm_config.py new file mode 100644 index 0000000..cf26a92 --- /dev/null +++ b/policy_module/osm_policy_module/common/alarm_config.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the Lcompletoicense at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## + + +class AlarmConfig: + def __init__(self, metric_name, vdu_name, vnf_member_index, threshold, operation, statistic, action): + self.metric_name = metric_name + self.vdu_name = vdu_name + self.vnf_member_index = vnf_member_index + self.threshold = threshold + self.operation = operation + self.statistic = statistic + self.action = action diff --git a/policy_module/osm_policy_module/common/lcm_client.py b/policy_module/osm_policy_module/common/lcm_client.py new file mode 100644 index 0000000..be44efe --- /dev/null +++ b/policy_module/osm_policy_module/common/lcm_client.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +import logging + +from kafka import KafkaProducer + +from osm_policy_module.core.config import Config + +log = logging.getLogger(__name__) + + +class LcmClient: + def __init__(self): + cfg = Config.instance() + self.kafka_server = '{}:{}'.format(cfg.get('policy_module', 'kafka_server_host'), + cfg.get('policy_module', 'kafka_server_port')) + self.producer = KafkaProducer(bootstrap_servers=self.kafka_server, + key_serializer=str.encode, + value_serializer=str.encode) + + def scale(self, nsr_id: str, name: str, action: str): + msg = self._create_scale_action_payload(nsr_id, name, action) + log.info("Sending scale action message: %s", json.dumps(msg)) + self.producer.send(topic='lcm_pm', key='trigger_scaling', value=json.dumps(msg)) + self.producer.flush() + + def _create_scale_action_payload(self, nsr_id: str, name: str, action: str): + msg = { + "ns_id": nsr_id, + "scaling_group_descriptor": { + "name": name, + "action": action + } + } + return msg diff --git a/policy_module/osm_policy_module/common/mon_client.py b/policy_module/osm_policy_module/common/mon_client.py new file mode 100644 index 0000000..d1336db --- /dev/null +++ b/policy_module/osm_policy_module/common/mon_client.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +import logging +import random +import uuid + +from kafka import KafkaProducer, KafkaConsumer + +from osm_policy_module.core.config import Config + +log = logging.getLogger(__name__) + + +class MonClient: + def __init__(self): + cfg = Config.instance() + self.kafka_server = '{}:{}'.format(cfg.get('policy_module', 'kafka_server_host'), + cfg.get('policy_module', 'kafka_server_port')) + self.producer = KafkaProducer(bootstrap_servers=self.kafka_server, + key_serializer=str.encode, + value_serializer=str.encode) + + def create_alarm(self, metric_name: str, ns_id: str, vdu_name: str, vnf_member_index: str, threshold: int, + statistic: str, operation: str): + cor_id = random.randint(1, 1000000) + msg = self._create_alarm_payload(cor_id, metric_name, ns_id, vdu_name, vnf_member_index, threshold, statistic, + operation) + log.info("Sending create_alarm_request %s", msg) + future = self.producer.send(topic='alarm_request', key='create_alarm_request', value=json.dumps(msg)) + future.get(timeout=60) + consumer = KafkaConsumer(bootstrap_servers=self.kafka_server, + key_deserializer=bytes.decode, + value_deserializer=bytes.decode, + consumer_timeout_ms=10000) + consumer.subscribe(['alarm_response']) + for message in consumer: + if message.key == 'create_alarm_response': + content = json.loads(message.value) + log.info("Received create_alarm_response %s", content) + if self._is_alarm_response_correlation_id_eq(cor_id, content): + alarm_uuid = content['alarm_create_response']['alarm_uuid'] + # TODO Handle error response + return alarm_uuid + + raise ValueError('Timeout: No alarm creation response from MON. Is MON up?') + + def _create_alarm_payload(self, cor_id: int, metric_name: str, ns_id: str, vdu_name: str, vnf_member_index: str, + threshold: int, statistic: str, operation: str): + alarm_create_request = { + 'correlation_id': cor_id, + 'alarm_name': str(uuid.uuid4()), + 'metric_name': metric_name, + 'ns_id': ns_id, + 'vdu_name': vdu_name, + 'vnf_member_index': vnf_member_index, + 'operation': operation, + 'severity': 'critical', + 'threshold_value': threshold, + 'statistic': statistic + } + msg = { + 'alarm_create_request': alarm_create_request, + } + return msg + + def _is_alarm_response_correlation_id_eq(self, cor_id, message_content): + return message_content['alarm_create_response']['correlation_id'] == cor_id diff --git a/policy_module/osm_policy_module/core/__init__.py b/policy_module/osm_policy_module/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/policy_module/osm_policy_module/core/agent.py b/policy_module/osm_policy_module/core/agent.py new file mode 100644 index 0000000..cdd5dfc --- /dev/null +++ b/policy_module/osm_policy_module/core/agent.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +import logging +from typing import Dict, List + +import yaml +from kafka import KafkaConsumer +from osm_policy_module.common.alarm_config import AlarmConfig +from osm_policy_module.common.lcm_client import LcmClient +from osm_policy_module.common.mon_client import MonClient +from osm_policy_module.core.config import Config +from osm_policy_module.core.database import ScalingRecord, ScalingAlarm + +log = logging.getLogger(__name__) + + +class PolicyModuleAgent: + def run(self): + cfg = Config.instance() + # Initialize servers + kafka_server = '{}:{}'.format(cfg.get('policy_module', 'kafka_server_host'), + cfg.get('policy_module', 'kafka_server_port')) + + # Initialize Kafka consumer + log.info("Connecting to Kafka server at %s", kafka_server) + consumer = KafkaConsumer(bootstrap_servers=kafka_server, + key_deserializer=bytes.decode, + value_deserializer=bytes.decode, + group_id="pm-consumer") + consumer.subscribe(['lcm_pm', 'alarm_response']) + + for message in consumer: + log.info("Message arrived: %s", message) + try: + if message.key == 'configure_scaling': + try: + content = json.loads(message.value) + except: + content = yaml.safe_load(message.value) + log.info("Creating scaling record in DB") + # TODO: Use transactions: http://docs.peewee-orm.com/en/latest/peewee/transactions.html + scaling_record = ScalingRecord.create( + nsr_id=content['ns_id'], + name=content['scaling_group_descriptor']['name'], + content=json.dumps(content) + ) + log.info("Created scaling record in DB : nsr_id=%s, name=%s, content=%s", + scaling_record.nsr_id, + scaling_record.name, + scaling_record.content) + alarm_configs = self._get_alarm_configs(content) + for config in alarm_configs: + mon_client = MonClient() + log.info("Creating alarm record in DB") + alarm_uuid = mon_client.create_alarm( + metric_name=config.metric_name, + ns_id=scaling_record.nsr_id, + vdu_name=config.vdu_name, + vnf_member_index=config.vnf_member_index, + threshold=config.threshold, + operation=config.operation, + statistic=config.statistic + ) + ScalingAlarm.create( + alarm_id=alarm_uuid, + action=config.action, + scaling_record=scaling_record + ) + if message.key == 'notify_alarm': + content = json.loads(message.value) + alarm_id = content['notify_details']['alarm_uuid'] + metric_name = content['notify_details']['metric_name'] + operation = content['notify_details']['operation'] + threshold = content['notify_details']['threshold_value'] + vdu_name = content['notify_details']['vdu_name'] + vnf_member_index = content['notify_details']['vnf_member_index'] + ns_id = content['notify_details']['ns_id'] + log.info( + "Received alarm notification for alarm %s, \ + metric %s, \ + operation %s, \ + threshold %s, \ + vdu_name %s, \ + vnf_member_index %s, \ + ns_id %s ", + alarm_id, metric_name, operation, threshold, vdu_name, vnf_member_index, ns_id) + try: + alarm = ScalingAlarm.select().where(ScalingAlarm.alarm_id == alarm_id).get() + lcm_client = LcmClient() + log.info("Sending scaling action message for ns: %s", alarm_id) + lcm_client.scale(alarm.scaling_record.nsr_id, alarm.scaling_record.name, alarm.action) + except ScalingAlarm.DoesNotExist: + log.info("There is no action configured for alarm %s.", alarm_id) + except Exception: + log.exception("Error consuming message: ") + + def _get_alarm_configs(self, message_content: Dict) -> List[AlarmConfig]: + scaling_criterias = message_content['scaling_group_descriptor']['scaling_policy']['scaling_criteria'] + alarm_configs = [] + for criteria in scaling_criterias: + metric_name = '' + scale_out_threshold = criteria['scale_out_threshold'] + scale_in_threshold = criteria['scale_in_threshold'] + scale_out_operation = criteria['scale_out_relational_operation'] + scale_in_operation = criteria['scale_in_relational_operation'] + statistic = criteria['monitoring_param']['aggregation_type'] + vdu_name = '' + vnf_member_index = '' + if 'vdu_monitoring_param' in criteria['monitoring_param']: + vdu_name = criteria['monitoring_param']['vdu_monitoring_param']['vdu_name'] + vnf_member_index = criteria['monitoring_param']['vdu_monitoring_param']['vnf_member_index'] + metric_name = criteria['monitoring_param']['vdu_monitoring_param']['name'] + if 'vnf_metric' in criteria['monitoring_param']: + # TODO vnf_metric + continue + if 'vdu_metric' in criteria['monitoring_param']: + # TODO vdu_metric + continue + scale_out_alarm_config = AlarmConfig(metric_name, + vdu_name, + vnf_member_index, + scale_out_threshold, + scale_out_operation, + statistic, + 'scale_out') + scale_in_alarm_config = AlarmConfig(metric_name, + vdu_name, + vnf_member_index, + scale_in_threshold, + scale_in_operation, + statistic, + 'scale_in') + alarm_configs.append(scale_in_alarm_config) + alarm_configs.append(scale_out_alarm_config) + return alarm_configs diff --git a/policy_module/osm_policy_module/core/config.py b/policy_module/osm_policy_module/core/config.py new file mode 100644 index 0000000..9899009 --- /dev/null +++ b/policy_module/osm_policy_module/core/config.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +##"""Global Configuration.""" + +import logging + +from osm_policy_module.core.singleton import Singleton + +try: + from configparser import ConfigParser +except ImportError: + from ConfigParser import ConfigParser + +log = logging.getLogger(__name__) + + +@Singleton +class Config(object): + """Global configuration.""" + + def __init__(self): + # Default config values + self.config = { + 'policy_module': { + 'kafka_server_host': '127.0.0.1', + 'kafka_server_port': '9092', + 'log_dir': 'stdout', + 'log_level': 'INFO', + 'enable_logstash_handler': 'false', + 'logstash_host': 'logstash', + 'logstash_port': '5000' + }, + } + + def load_file(self, config_file_path): + if config_file_path: + config_parser = ConfigParser() + config_parser.read(config_file_path) + for section in config_parser.sections(): + for key, value in config_parser.items(section): + if section not in self.config: + self.config[section] = {} + self.config[section][key] = value + + def get(self, group, name=None, default=None): + if group in self.config: + if name is None: + return self.config[group] + return self.config[group].get(name, default) + return default + + def __str__(self): + return str(self.config) diff --git a/policy_module/osm_policy_module/core/database.py b/policy_module/osm_policy_module/core/database.py new file mode 100644 index 0000000..0757b7b --- /dev/null +++ b/policy_module/osm_policy_module/core/database.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import logging + +from peewee import * +from playhouse.sqlite_ext import SqliteExtDatabase + +from osm_policy_module.core.config import Config + +log = logging.getLogger(__name__) +cfg = Config.instance() + +db = SqliteExtDatabase('policy_module.db') + + +class BaseModel(Model): + class Meta: + database = db + + +class ScalingRecord(BaseModel): + nsr_id = CharField() + name = CharField() + content = TextField() + + +class ScalingAlarm(BaseModel): + alarm_id = CharField() + action = CharField() + scaling_record = ForeignKeyField(ScalingRecord, related_name='scaling_alarms') + + +class DatabaseManager: + def create_tables(self): + try: + db.connect() + db.create_tables([ScalingRecord, ScalingAlarm]) + db.close() + except Exception as e: + log.exception("Error creating tables: ") diff --git a/policy_module/osm_policy_module/core/singleton.py b/policy_module/osm_policy_module/core/singleton.py new file mode 100644 index 0000000..9fb571b --- /dev/null +++ b/policy_module/osm_policy_module/core/singleton.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +##"""Simple singleton class.""" + +from __future__ import unicode_literals + + +class Singleton(object): + """Simple singleton class.""" + + def __init__(self, decorated): + """Initialize singleton instance.""" + self._decorated = decorated + + def instance(self): + """Return singleton instance.""" + try: + return self._instance + except AttributeError: + self._instance = self._decorated() + return self._instance diff --git a/policy_module/osm_policy_module/models/configure_scaling.json b/policy_module/osm_policy_module/models/configure_scaling.json new file mode 100644 index 0000000..0a87491 --- /dev/null +++ b/policy_module/osm_policy_module/models/configure_scaling.json @@ -0,0 +1,111 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "ns_id": { + "type": "string" + }, + "scaling_group_descriptor": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "scaling_policy": { + "type": "object", + "properties": { + "scale_in_operation_type": { + "type": "string", + "enum": [ + "and", + "or" + ] + }, + "scale_out_operation_type": { + "type": "string", + "enum": [ + "and", + "or" + ] + }, + "threshold_time": { + "type": "number" + }, + "cooldown_time": { + "type": "number" + }, + "scaling_criteria": { + "type": "array", + "items": { + "type": "object", + "properties": { + "scale_in_threshold": { + "type": "number" + }, + "scale_out_threshold": { + "type": "number" + }, + "scale_in_relational_operation": { + "type": "string", + "enum": [ + "lt", + "gt", + "le", + "ge", + "eq" + ] + }, + "scale_out_relational_operation": { + "type": "string", + "enum": [ + "lt", + "gt", + "le", + "ge", + "eq" + ] + }, + "monitoring_param": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "aggregation_type": { + "type": "string", + "enum": [ + "average", + "maximum", + "minimum", + "count", + "sum" + ] + }, + "vdu_monitoring_param": { + "type": "object", + "properties": { + "vnf_member_index": { + "type": "string" + }, + "vdu_name": { + "type": "string" + }, + "name": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/policy_module/osm_policy_module/tests/__init__.py b/policy_module/osm_policy_module/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/policy_module/osm_policy_module/tests/examples/configure_scaling_full_example.json b/policy_module/osm_policy_module/tests/examples/configure_scaling_full_example.json new file mode 100644 index 0000000..eab1cc7 --- /dev/null +++ b/policy_module/osm_policy_module/tests/examples/configure_scaling_full_example.json @@ -0,0 +1,30 @@ +{ + "ns_id": "360b400b-86dc-4b8e-a139-b7fc3987cf69", + "scaling_group_descriptor": { + "name": "test", + "scaling_policy": { + "scale_in_operation_type": "or", + "scale_out_operation_type": "or", + "threshold_time": 10, + "cooldown_time": 10, + "scaling_criteria": [ + { + "scale_in_threshold": 50, + "scale_out_threshold": 50, + "scale_in_relational_operation": "lt", + "scale_out_relational_operation": "gt", + "monitoring_param": { + "id": "test_param_id", + "name": "test_param", + "aggregation_type": "average", + "vdu_monitoring_param": { + "vnf_member_index": "1", + "vdu_name": "2d8d5355-acf7-42be-9f34-a10d02f9df39", + "name": "cpu_utilization" + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/policy_module/osm_policy_module/tests/integration/__init__.py b/policy_module/osm_policy_module/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/policy_module/osm_policy_module/tests/integration/test_scaling_config_kafka_msg.py b/policy_module/osm_policy_module/tests/integration/test_scaling_config_kafka_msg.py new file mode 100644 index 0000000..aea3f4a --- /dev/null +++ b/policy_module/osm_policy_module/tests/integration/test_scaling_config_kafka_msg.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +import logging +import os +import unittest + +from kafka import KafkaProducer, KafkaConsumer +from kafka.errors import KafkaError + +log = logging.getLogger(__name__) + + +class ScalingConfigTest(unittest.TestCase): + def setUp(self): + try: + kafka_server = '{}:{}'.format(os.getenv("KAFKA_SERVER_HOST", "localhost"), + os.getenv("KAFKA_SERVER_PORT", "9092")) + self.producer = KafkaProducer(bootstrap_servers=kafka_server, + key_serializer=str.encode, + value_serializer=str.encode) + self.consumer = KafkaConsumer(bootstrap_servers='localhost:9092', + group_id='osm_mon') + self.consumer.subscribe(['lcm_pm']) + except KafkaError: + self.skipTest('Kafka server not present.') + + def test_send_scaling_config_msg(self): + try: + with open( + os.path.join(os.path.dirname(__file__), '../examples/configure_scaling_full_example.json')) as file: + payload = json.load(file) + future = self.producer.send('lcm_pm', json.dumps(payload), key="configure_scaling") + result = future.get(timeout=60) + log.info('Result: %s', result) + + self.producer.flush() + # TODO: Improve assertions + self.assertIsNotNone(result) + except Exception as e: + self.fail(e) + + +if __name__ == '__main__': + unittest.main() diff --git a/policy_module/osm_policy_module/tests/unit/__init__.py b/policy_module/osm_policy_module/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/policy_module/osm_policy_module/tests/unit/test_examples.py b/policy_module/osm_policy_module/tests/unit/test_examples.py new file mode 100644 index 0000000..97b3370 --- /dev/null +++ b/policy_module/osm_policy_module/tests/unit/test_examples.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +import unittest + +import os + +from jsonschema import validate + + +class ExamplesTest(unittest.TestCase): + def test_examples_schema(self): + example_file_path = os.path.join(os.path.dirname(__file__), '../examples/configure_scaling_full_example.json') + schema_file_path = os.path.join(os.path.dirname(__file__), '../../models/configure_scaling.json') + with open(example_file_path) as example_file, open(schema_file_path) as schema_file: + validate(json.load(example_file), json.load(schema_file)) + + +if __name__ == '__main__': + unittest.main() diff --git a/policy_module/osm_policy_module/tests/unit/test_policy_agent.py b/policy_module/osm_policy_module/tests/unit/test_policy_agent.py new file mode 100644 index 0000000..0e12e84 --- /dev/null +++ b/policy_module/osm_policy_module/tests/unit/test_policy_agent.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +import os +import unittest + +from osm_policy_module.core.agent import PolicyModuleAgent + + +class PolicyAgentTest(unittest.TestCase): + def setUp(self): + self.agent = PolicyModuleAgent() + + def test_get_alarm_configs(self): + with open(os.path.join(os.path.dirname(__file__), '../examples/configure_scaling_full_example.json')) as file: + example = json.load(file) + alarm_configs = self.agent._get_alarm_configs(example) + # TODO Improve assertions + self.assertEqual(len(alarm_configs), 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/policy_module/requirements.txt b/policy_module/requirements.txt new file mode 100644 index 0000000..ad2c105 --- /dev/null +++ b/policy_module/requirements.txt @@ -0,0 +1,7 @@ +kafka==1.3.* +peewee==3.1.* +jsonschema==2.6.* +six==1.11.* +pyyaml==3.* +python-logstash==0.4.* +git+https://osm.etsi.org/gerrit/osm/common.git@v4.0.1#egg=osm-common \ No newline at end of file diff --git a/policy_module/scripts/gen_config_from_env.sh b/policy_module/scripts/gen_config_from_env.sh new file mode 100644 index 0000000..3dcaf67 --- /dev/null +++ b/policy_module/scripts/gen_config_from_env.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +CONFIG_FILENAME="osm_policy_agent.cfg" +rm $CONFIG_FILENAME 2> /dev/null +touch $CONFIG_FILENAME +echo "[policy_module]" >> $CONFIG_FILENAME +if ! [[ -z "${BROKER_URI}" ]]; then + HOST=$(echo $BROKER_URI | cut -d: -f1) + PORT=$(echo $BROKER_URI | cut -d: -f2) + echo "kafka_server_host=$HOST" >> $CONFIG_FILENAME + echo "kafka_server_port=$PORT" >> $CONFIG_FILENAME +fi +if ! [[ -z "${LOGSTASH_URI}" ]]; then + HOST=$(echo $LOGSTASH_URI | cut -d: -f1) + PORT=$(echo $LOGSTASH_URI | cut -d: -f2) + echo "enable_logstash_handler=true" >> $CONFIG_FILENAME + echo "logstash_host=$HOST" >> $CONFIG_FILENAME + echo "logstash_port=$PORT" >> $CONFIG_FILENAME +fi \ No newline at end of file diff --git a/policy_module/setup.py b/policy_module/setup.py new file mode 100644 index 0000000..d5c3562 --- /dev/null +++ b/policy_module/setup.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import setuptools + + +def parse_requirements(requirements): + with open(requirements) as f: + return [l.strip('\n') for l in f if l.strip('\n') and not l.startswith('#') and '://' not in l] + + +_author = "Benjamín Díaz" +_name = 'osm_policy_module' +_author_email = 'bdiaz@whitestack.com' +_version = '1.0' +_description = 'OSM Policy Module' +_maintainer = 'Benjamín Díaz' +_maintainer_email = 'bdiaz@whitestack.com' +_license = 'Apache 2.0' +_url = 'https://osm.etsi.org/gitweb/?p=osm/MON.git;a=tree' + +setuptools.setup( + name=_name, + version=_version, + description=_description, + long_description=open('README.rst').read(), + author=_author, + author_email=_author_email, + maintainer=_maintainer, + maintainer_email=_maintainer_email, + url=_url, + license=_license, + packages=setuptools.find_packages(), + include_package_data=True, + install_requires=[ + 'kafka==1.3.*', + 'peewee==3.1.*', + 'jsonschema==2.6.*', + 'six==1.11.*', + 'pyyaml==3.*', + 'python-logstash==0.4.*', + 'osm-common==4.*' + ], + entry_points={ + "console_scripts": [ + "osm-policy-agent = osm_policy_module.cmd.policy_module_agent:main", + ] + }, + dependency_links=[ + 'git+https://osm.etsi.org/gerrit/osm/common.git@v4.0.1#egg=osm-common' + ] +) -- 2.25.1