From 7f11ecff803667fb5cd0e79389eece83ddc96c86 Mon Sep 17 00:00:00 2001 From: Benjamin Diaz Date: Fri, 14 Sep 2018 12:03:38 -0300 Subject: [PATCH] Migrates POL code from MON repo Adds support for VDU metric autoscaling Modifies env var names Signed-off-by: Benjamin Diaz Change-Id: If9587e1b8eacaf6fb297306050a97d33c8a63ead --- .gitignore | 78 +++ Dockerfile | 28 ++ Jenkinsfile | 32 ++ MANIFEST.in | 28 ++ README.rst | 13 + devops-stages/stage-archive.sh | 34 ++ devops-stages/stage-build.sh | 29 ++ devops-stages/stage-test.sh | 26 + docker/Dockerfile | 51 ++ osm_policy_module/__init__.py | 0 osm_policy_module/cmd/__init__.py | 0 osm_policy_module/cmd/policy_module_agent.py | 63 +++ osm_policy_module/common/__init__.py | 0 osm_policy_module/common/lcm_client.py | 89 ++++ osm_policy_module/common/mon_client.py | 89 ++++ osm_policy_module/core/__init__.py | 0 osm_policy_module/core/agent.py | 205 ++++++++ osm_policy_module/core/config.py | 89 ++++ osm_policy_module/core/database.py | 63 +++ osm_policy_module/core/singleton.py | 42 ++ osm_policy_module/tests/__init__.py | 0 .../examples/cirros_vdu_scaling_nsd.yaml | 42 ++ .../examples/cirros_vdu_scaling_vnfd.yaml | 74 +++ .../tests/examples/instantiated.json | 4 + .../tests/integration/__init__.py | 0 .../tests/integration/test_kafka_messages.py | 79 +++ .../tests/integration/test_policy_agent.py | 456 ++++++++++++++++++ osm_policy_module/tests/unit/__init__.py | 0 .../tests/unit/test_policy_agent.py | 35 ++ requirements.txt | 6 + setup.py | 66 +++ test-requirements.txt | 23 + tox.ini | 57 +++ 33 files changed, 1801 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Jenkinsfile create mode 100644 MANIFEST.in create mode 100644 README.rst create mode 100755 devops-stages/stage-archive.sh create mode 100755 devops-stages/stage-build.sh create mode 100755 devops-stages/stage-test.sh create mode 100644 docker/Dockerfile create mode 100644 osm_policy_module/__init__.py create mode 100644 osm_policy_module/cmd/__init__.py create mode 100644 osm_policy_module/cmd/policy_module_agent.py create mode 100644 osm_policy_module/common/__init__.py create mode 100644 osm_policy_module/common/lcm_client.py create mode 100644 osm_policy_module/common/mon_client.py create mode 100644 osm_policy_module/core/__init__.py create mode 100644 osm_policy_module/core/agent.py create mode 100644 osm_policy_module/core/config.py create mode 100644 osm_policy_module/core/database.py create mode 100644 osm_policy_module/core/singleton.py create mode 100644 osm_policy_module/tests/__init__.py create mode 100644 osm_policy_module/tests/examples/cirros_vdu_scaling_nsd.yaml create mode 100644 osm_policy_module/tests/examples/cirros_vdu_scaling_vnfd.yaml create mode 100644 osm_policy_module/tests/examples/instantiated.json create mode 100644 osm_policy_module/tests/integration/__init__.py create mode 100644 osm_policy_module/tests/integration/test_kafka_messages.py create mode 100644 osm_policy_module/tests/integration/test_policy_agent.py create mode 100644 osm_policy_module/tests/unit/__init__.py create mode 100644 osm_policy_module/tests/unit/test_policy_agent.py create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 test-requirements.txt create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f4d6bb1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,78 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: helena.mcgough@intel.com or adrian.hoban@intel.com +## +*.py[cod] + +# C extensions +*.so + +# log files +*.log + +# Packages +*.egg +*.egg-info +dist +build +.eggs +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +nohup.out + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.testrepository +.venv +.cache + +# Translations +*.mo + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.*sw? +.settings/ +__pycache__/ +.idea diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6738633 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## + +FROM ubuntu:16.04 +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install git tox make python python-pip python3 python3-pip debhelper && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install wget python-dev python-software-properties python-stdeb && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install default-jre libmysqlclient-dev && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install libmysqlclient-dev libxml2 python3-all diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..6a384d5 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,32 @@ +properties([ + parameters([ + string(defaultValue: env.BRANCH_NAME, description: '', name: 'GERRIT_BRANCH'), + string(defaultValue: 'osm/POL', description: '', name: 'GERRIT_PROJECT'), + string(defaultValue: env.GERRIT_REFSPEC, description: '', name: 'GERRIT_REFSPEC'), + string(defaultValue: env.GERRIT_PATCHSET_REVISION, description: '', name: 'GERRIT_PATCHSET_REVISION'), + string(defaultValue: 'https://osm.etsi.org/gerrit', description: '', name: 'PROJECT_URL_PREFIX'), + booleanParam(defaultValue: false, description: '', name: 'TEST_INSTALL'), + string(defaultValue: 'artifactory-osm', description: '', name: 'ARTIFACTORY_SERVER'), + ]) +]) + +def devops_checkout() { + dir('devops') { + git url: "${PROJECT_URL_PREFIX}/osm/devops", branch: params.GERRIT_BRANCH + } +} + +node { + checkout scm + devops_checkout() + + ci_helper = load "devops/jenkins/ci-pipelines/ci_stage_2.groovy" + ci_helper.ci_pipeline( 'POL', + params.PROJECT_URL_PREFIX, + params.GERRIT_PROJECT, + params.GERRIT_BRANCH, + params.GERRIT_REFSPEC, + params.GERRIT_PATCHSET_REVISION, + params.TEST_INSTALL, + params.ARTIFACTORY_SERVER) +} diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ad2c95a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,28 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## + +include requirements.txt +include test-requirements.txt +include README.rst +recursive-include osm_policy_module *.py *.xml *.sh +recursive-include devops-stages * +recursive-include test *.py diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..89b0322 --- /dev/null +++ b/README.rst @@ -0,0 +1,13 @@ +Install +------------------------ + :: + + git clone https://osm.etsi.org/gerrit/osm/POL.git + pip install ./POL + +Run +------------------------ + :: + + osm-policy-agent + diff --git a/devops-stages/stage-archive.sh b/devops-stages/stage-archive.sh new file mode 100755 index 0000000..394ea40 --- /dev/null +++ b/devops-stages/stage-archive.sh @@ -0,0 +1,34 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com + +#__author__ = "Prithiv Mohan" +#__date__ = "25/Sep/2017" + + +#!/bin/sh +rm -rf pool +rm -rf dists +mkdir -p pool/MON +mv deb_dist/*.deb pool/MON/ +mkdir -p dists/unstable/MON/binary-amd64/ +apt-ftparchive packages pool/MON > dists/unstable/MON/binary-amd64/Packages +gzip -9fk dists/unstable/MON/binary-amd64/Packages +echo 'dists/**,pool/MON/*.deb' \ No newline at end of file diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh new file mode 100755 index 0000000..4251b1c --- /dev/null +++ b/devops-stages/stage-build.sh @@ -0,0 +1,29 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com + +#__author__ = "Prithiv Mohan" +#__date__ = "14/Sep/2017" + +#!/bin/bash +rm -rf deb_dist +rm -rf dist +rm -rf osm_mon.egg-info +tox -e build diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh new file mode 100755 index 0000000..d588666 --- /dev/null +++ b/devops-stages/stage-test.sh @@ -0,0 +1,26 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: prithiv.mohan@intel.com or adrian.hoban@intel.com + +#__author__ = "Prithiv Mohan" +#__date__ = "14/Sep/2017" + +#!/bin/bash +tox diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..4095fa8 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,51 @@ +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## + +FROM ubuntu:16.04 + +LABEL authors="Benjamín Díaz" + +RUN apt-get --yes update \ + && apt-get --yes install python3 python3-pip libmysqlclient-dev git \ + && pip3 install pip==9.0.3 + +COPY requirements.txt /policy_module/requirements.txt + +RUN pip3 install -r /policy_module/requirements.txt + +COPY . /policy_module + +RUN pip3 install /policy_module + +ENV OSMPOL_MESSAGE_DRIVER kafka +ENV OSMPOL_MESSAGE_HOST kafka +ENV OSMPOL_MESSAGE_PORT 9092 + +ENV OSMPOL_DATABASE_DRIVER mongo +ENV OSMPOL_DATABASE_HOST mongo +ENV OSMPOL_DATABASE_PORT 27017 + +ENV OSMPOL_SQL_DATABASE_URI sqlite:///mon_sqlite.db + +ENV OSMPOL_LOG_LEVEL INFO + +CMD osm-policy-agent diff --git a/osm_policy_module/__init__.py b/osm_policy_module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/osm_policy_module/cmd/__init__.py b/osm_policy_module/cmd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/osm_policy_module/cmd/policy_module_agent.py b/osm_policy_module/cmd/policy_module_agent.py new file mode 100644 index 0000000..24663e5 --- /dev/null +++ b/osm_policy_module/cmd/policy_module_agent.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import argparse +import logging +import sys + +from osm_policy_module.core.agent import PolicyModuleAgent +from osm_policy_module.core.config import Config +from osm_policy_module.core.database import DatabaseManager + + +def main(): + cfg = Config.instance() + parser = argparse.ArgumentParser(prog='pm-scaling-config-agent') + parser.add_argument('--config-file', nargs='?', help='Policy module agent configuration file') + args = parser.parse_args() + if args.config_file: + cfg.load_file(args.config_file) + log_formatter_str = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + logging.basicConfig(stream=sys.stdout, + format=log_formatter_str, + datefmt='%m/%d/%Y %I:%M:%S %p', + level=logging.getLevelName(cfg.OSMPOL_LOG_LEVEL)) + kafka_logger = logging.getLogger('kafka') + kafka_logger.setLevel(logging.WARN) + kafka_formatter = logging.Formatter(log_formatter_str) + kafka_handler = logging.StreamHandler(sys.stdout) + kafka_handler.setFormatter(kafka_formatter) + kafka_logger.addHandler(kafka_handler) + log = logging.getLogger(__name__) + log.info("Config: %s", cfg) + log.info("Syncing database...") + db_manager = DatabaseManager() + db_manager.create_tables() + log.info("Database synced correctly.") + log.info("Starting policy module agent...") + agent = PolicyModuleAgent() + agent.run() + + +if __name__ == '__main__': + main() diff --git a/osm_policy_module/common/__init__.py b/osm_policy_module/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/osm_policy_module/common/lcm_client.py b/osm_policy_module/common/lcm_client.py new file mode 100644 index 0000000..7857d26 --- /dev/null +++ b/osm_policy_module/common/lcm_client.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import datetime +import json +import logging +import time +import uuid + +from kafka import KafkaProducer +from osm_common import dbmongo + +from osm_policy_module.core.config import Config + +log = logging.getLogger(__name__) + + +class LcmClient: + def __init__(self): + cfg = Config.instance() + self.kafka_server = '{}:{}'.format(cfg.OSMPOL_MESSAGE_HOST, + cfg.OSMPOL_MESSAGE_PORT) + self.producer = KafkaProducer(bootstrap_servers=self.kafka_server, + key_serializer=str.encode, + value_serializer=str.encode) + self.common_db = dbmongo.DbMongo() + self.common_db.db_connect({'host': cfg.OSMPOL_DATABASE_HOST, + 'port': int(cfg.OSMPOL_DATABASE_PORT), + 'name': 'osm'}) + + def scale(self, nsr_id: str, scaling_group_name: str, vnf_member_index: int, action: str): + nslcmop = self._generate_nslcmop(nsr_id, scaling_group_name, vnf_member_index, action) + self.common_db.create("nslcmops", nslcmop) + log.info("Sending scale action message: %s", json.dumps(nslcmop)) + self.producer.send(topic='ns', key='scale', value=json.dumps(nslcmop)) + self.producer.flush() + + def _generate_nslcmop(self, nsr_id: str, scaling_group_name: str, vnf_member_index: int, action: str): + _id = str(uuid.uuid4()) + now = time.time() + params = { + "scaleType": "SCALE_VNF", + "scaleVnfData": { + "scaleVnfType": action.upper(), + "scaleByStepData": { + "scaling-group-descriptor": scaling_group_name, + "member-vnf-index": str(vnf_member_index) + } + }, + "scaleTime": "{}Z".format(datetime.datetime.utcnow().isoformat()) + } + + nslcmop = { + "id": _id, + "_id": _id, + "operationState": "PROCESSING", + "statusEnteredTime": now, + "nsInstanceId": nsr_id, + "lcmOperationType": "scale", + "startTime": now, + "isAutomaticInvocation": True, + "operationParams": params, + "isCancelPending": False, + "links": { + "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id, + "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id, + } + } + return nslcmop diff --git a/osm_policy_module/common/mon_client.py b/osm_policy_module/common/mon_client.py new file mode 100644 index 0000000..724fe83 --- /dev/null +++ b/osm_policy_module/common/mon_client.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +import logging +import random +import uuid + +from kafka import KafkaProducer, KafkaConsumer + +from osm_policy_module.core.config import Config + +log = logging.getLogger(__name__) + + +class MonClient: + def __init__(self): + cfg = Config.instance() + self.kafka_server = '{}:{}'.format(cfg.OSMPOL_MESSAGE_HOST, + cfg.OSMPOL_MESSAGE_PORT) + self.producer = KafkaProducer(bootstrap_servers=self.kafka_server, + key_serializer=str.encode, + value_serializer=str.encode) + + def create_alarm(self, metric_name: str, ns_id: str, vdu_name: str, vnf_member_index: int, threshold: int, + statistic: str, operation: str): + cor_id = random.randint(1, 1000000) + msg = self._create_alarm_payload(cor_id, metric_name, ns_id, vdu_name, vnf_member_index, threshold, statistic, + operation) + log.info("Sending create_alarm_request %s", msg) + self.producer.send(topic='alarm_request', key='create_alarm_request', value=json.dumps(msg)) + self.producer.flush() + consumer = KafkaConsumer(bootstrap_servers=self.kafka_server, + key_deserializer=bytes.decode, + value_deserializer=bytes.decode, + consumer_timeout_ms=10000) + consumer.subscribe(['alarm_response']) + for message in consumer: + if message.key == 'create_alarm_response': + content = json.loads(message.value) + log.info("Received create_alarm_response %s", content) + if self._is_alarm_response_correlation_id_eq(cor_id, content): + alarm_uuid = content['alarm_create_response']['alarm_uuid'] + # TODO Handle error response + return alarm_uuid + + raise ValueError('Timeout: No alarm creation response from MON. Is MON up?') + + def _create_alarm_payload(self, cor_id: int, metric_name: str, ns_id: str, vdu_name: str, vnf_member_index: int, + threshold: int, statistic: str, operation: str): + alarm_create_request = { + 'correlation_id': cor_id, + 'alarm_name': str(uuid.uuid4()), + 'metric_name': metric_name, + 'ns_id': ns_id, + 'vdu_name': vdu_name, + 'vnf_member_index': vnf_member_index, + 'operation': operation, + 'severity': 'critical', + 'threshold_value': threshold, + 'statistic': statistic + } + msg = { + 'alarm_create_request': alarm_create_request, + } + return msg + + def _is_alarm_response_correlation_id_eq(self, cor_id, message_content): + return message_content['alarm_create_response']['correlation_id'] == cor_id diff --git a/osm_policy_module/core/__init__.py b/osm_policy_module/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/osm_policy_module/core/agent.py b/osm_policy_module/core/agent.py new file mode 100644 index 0000000..8309da1 --- /dev/null +++ b/osm_policy_module/core/agent.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +import logging +import threading +from json import JSONDecodeError + +import yaml +from kafka import KafkaConsumer +from osm_common import dbmongo + +from osm_policy_module.common.lcm_client import LcmClient +from osm_policy_module.common.mon_client import MonClient +from osm_policy_module.core import database +from osm_policy_module.core.config import Config +from osm_policy_module.core.database import ScalingRecord, ScalingAlarm + +log = logging.getLogger(__name__) + + +class PolicyModuleAgent: + def __init__(self): + cfg = Config.instance() + self.common_db = dbmongo.DbMongo() + self.common_db.db_connect({'host': cfg.OSMPOL_DATABASE_HOST, + 'port': int(cfg.OSMPOL_DATABASE_PORT), + 'name': 'osm'}) + self.mon_client = MonClient() + self.kafka_server = '{}:{}'.format(cfg.OSMPOL_MESSAGE_HOST, + cfg.OSMPOL_MESSAGE_PORT) + + def run(self): + cfg = Config.instance() + cfg.read_environ() + + consumer = KafkaConsumer(bootstrap_servers=self.kafka_server, + key_deserializer=bytes.decode, + value_deserializer=bytes.decode, + consumer_timeout_ms=10000) + consumer.subscribe(["ns", "alarm_response"]) + + for message in consumer: + t = threading.Thread(target=self._process_msg, args=(message.topic, message.key, message.value,)) + t.start() + + def _process_msg(self, topic, key, msg): + try: + # Check for ns instantiation + if key == 'instantiated': + try: + content = json.loads(msg) + except JSONDecodeError: + content = yaml.safe_load(msg) + log.info("Message arrived with topic: %s, key: %s, msg: %s", topic, key, content) + nslcmop_id = content['nslcmop_id'] + nslcmop = self.common_db.get_one(table="nslcmops", + filter={"_id": nslcmop_id}) + if nslcmop['operationState'] == 'COMPLETED' or nslcmop['operationState'] == 'PARTIALLY_COMPLETED': + nsr_id = nslcmop['nsInstanceId'] + log.info("Configuring scaling groups for network service with nsr_id: %s", nsr_id) + self._configure_scaling_groups(nsr_id) + else: + log.info( + "Network service is not in COMPLETED or PARTIALLY_COMPLETED state. " + "Current state is %s. Skipping...", + nslcmop['operationState']) + + if key == 'notify_alarm': + try: + content = json.loads(msg) + except JSONDecodeError: + content = yaml.safe_load(msg) + log.info("Message arrived with topic: %s, key: %s, msg: %s", topic, key, content) + alarm_id = content['notify_details']['alarm_uuid'] + metric_name = content['notify_details']['metric_name'] + operation = content['notify_details']['operation'] + threshold = content['notify_details']['threshold_value'] + vdu_name = content['notify_details']['vdu_name'] + vnf_member_index = content['notify_details']['vnf_member_index'] + ns_id = content['notify_details']['ns_id'] + log.info( + "Received alarm notification for alarm %s, \ + metric %s, \ + operation %s, \ + threshold %s, \ + vdu_name %s, \ + vnf_member_index %s, \ + ns_id %s ", + alarm_id, metric_name, operation, threshold, vdu_name, vnf_member_index, ns_id) + try: + alarm = ScalingAlarm.select().where(ScalingAlarm.alarm_id == alarm_id).get() + lcm_client = LcmClient() + log.info("Sending scaling action message for ns: %s", alarm_id) + lcm_client.scale(alarm.scaling_record.nsr_id, alarm.scaling_record.name, alarm.vnf_member_index, + alarm.action) + except ScalingAlarm.DoesNotExist: + log.info("There is no action configured for alarm %s.", alarm_id) + except Exception: + log.exception("Error consuming message: ") + + def _get_vnfr(self, nsr_id: str, member_index: int): + vnfr = self.common_db.get_one(table="vnfrs", + filter={"nsr-id-ref": nsr_id, "member-vnf-index-ref": str(member_index)}) + return vnfr + + def _get_vnfrs(self, nsr_id: str): + return [self._get_vnfr(nsr_id, member['member-vnf-index']) for member in + self._get_nsr(nsr_id)['nsd']['constituent-vnfd']] + + def _get_vnfd(self, vnfd_id: str): + vnfr = self.common_db.get_one(table="vnfds", + filter={"_id": vnfd_id}) + return vnfr + + def _get_nsr(self, nsr_id: str): + nsr = self.common_db.get_one(table="nsrs", + filter={"id": nsr_id}) + return nsr + + def _configure_scaling_groups(self, nsr_id: str): + # TODO(diazb): Check for alarm creation on exception and clean resources if needed. + with database.db.atomic(): + vnfrs = self._get_vnfrs(nsr_id) + log.info("Checking %s vnfrs...", len(vnfrs)) + for vnfr in vnfrs: + vnfd = self._get_vnfd(vnfr['vnfd-id']) + log.info("Looking for vnfd %s", vnfr['vnfd-id']) + scaling_groups = vnfd['scaling-group-descriptor'] + vnf_monitoring_params = vnfd['monitoring-param'] + for scaling_group in scaling_groups: + log.info("Creating scaling record in DB...") + scaling_record = ScalingRecord.create( + nsr_id=nsr_id, + name=scaling_group['name'], + content=json.dumps(scaling_group) + ) + log.info("Created scaling record in DB : nsr_id=%s, name=%s, content=%s", + scaling_record.nsr_id, + scaling_record.name, + scaling_record.content) + for scaling_policy in scaling_group['scaling-policy']: + for vdur in vnfd['vdu']: + vdu_monitoring_params = vdur['monitoring-param'] + for scaling_criteria in scaling_policy['scaling-criteria']: + vnf_monitoring_param = next( + filter(lambda param: param['id'] == scaling_criteria['vnf-monitoring-param-ref'], + vnf_monitoring_params)) + # TODO: Add support for non-nfvi metrics + vdu_monitoring_param = next( + filter( + lambda param: param['id'] == vnf_monitoring_param['vdu-monitoring-param-ref'], + vdu_monitoring_params)) + alarm_uuid = self.mon_client.create_alarm( + metric_name=vdu_monitoring_param['nfvi-metric'], + ns_id=nsr_id, + vdu_name=vdur['name'], + vnf_member_index=vnfr['member-vnf-index-ref'], + threshold=scaling_criteria['scale-in-threshold'], + operation=scaling_criteria['scale-in-relational-operation'], + statistic=vnf_monitoring_param['aggregation-type'] + ) + ScalingAlarm.create( + alarm_id=alarm_uuid, + action='scale_in', + vnf_member_index=int(vnfr['member-vnf-index-ref']), + vdu_name=vdur['name'], + scaling_record=scaling_record + ) + alarm_uuid = self.mon_client.create_alarm( + metric_name=vdu_monitoring_param['nfvi-metric'], + ns_id=nsr_id, + vdu_name=vdur['name'], + vnf_member_index=vnfr['member-vnf-index-ref'], + threshold=scaling_criteria['scale-out-threshold'], + operation=scaling_criteria['scale-out-relational-operation'], + statistic=vnf_monitoring_param['aggregation-type'] + ) + ScalingAlarm.create( + alarm_id=alarm_uuid, + action='scale_out', + vnf_member_index=int(vnfr['member-vnf-index-ref']), + vdu_name=vdur['name'], + scaling_record=scaling_record + ) diff --git a/osm_policy_module/core/config.py b/osm_policy_module/core/config.py new file mode 100644 index 0000000..dab409b --- /dev/null +++ b/osm_policy_module/core/config.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +"""Global configuration managed by environment variables.""" + +import logging +import os + +from collections import namedtuple + +import six + +from osm_policy_module.core.singleton import Singleton + +log = logging.getLogger(__name__) + + +class BadConfigError(Exception): + """Configuration exception.""" + + pass + + +class CfgParam(namedtuple('CfgParam', ['key', 'default', 'data_type'])): + """Configuration parameter definition.""" + + def value(self, data): + """Convert a string to the parameter type.""" + try: + return self.data_type(data) + except (ValueError, TypeError): + raise BadConfigError( + 'Invalid value "%s" for configuration parameter "%s"' % ( + data, self.key)) + + +@Singleton +class Config(object): + """Configuration object.""" + + _configuration = [ + CfgParam('OSMPOL_MESSAGE_DRIVER', "kafka", six.text_type), + CfgParam('OSMPOL_MESSAGE_HOST', "localhost", six.text_type), + CfgParam('OSMPOL_MESSAGE_PORT', 9092, int), + CfgParam('OSMPOL_DATABASE_DRIVER', "mongo", six.text_type), + CfgParam('OSMPOL_DATABASE_HOST', "mongo", six.text_type), + CfgParam('OSMPOL_DATABASE_PORT', 27017, int), + CfgParam('OSMPOL_SQL_DATABASE_URI', "sqlite:///mon_sqlite.db", six.text_type), + CfgParam('OSMPOL_LOG_LEVEL', "INFO", six.text_type), + ] + + _config_dict = {cfg.key: cfg for cfg in _configuration} + _config_keys = _config_dict.keys() + + def __init__(self): + """Set the default values.""" + for cfg in self._configuration: + setattr(self, cfg.key, cfg.default) + self.read_environ() + + def read_environ(self): + """Check the appropriate environment variables and update defaults.""" + for key in self._config_keys: + try: + val = self._config_dict[key].data_type(os.environ[key]) + setattr(self, key, val) + except KeyError as exc: + log.debug("Environment variable not present: %s", exc) + return diff --git a/osm_policy_module/core/database.py b/osm_policy_module/core/database.py new file mode 100644 index 0000000..a89baed --- /dev/null +++ b/osm_policy_module/core/database.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import logging + +from peewee import * +from playhouse.sqlite_ext import SqliteExtDatabase + +from osm_policy_module.core.config import Config + +log = logging.getLogger(__name__) +cfg = Config.instance() + +db = SqliteExtDatabase('policy_module.db') + + +class BaseModel(Model): + class Meta: + database = db + + +class ScalingRecord(BaseModel): + nsr_id = CharField() + name = CharField() + content = TextField() + + +class ScalingAlarm(BaseModel): + alarm_id = CharField() + action = CharField() + vnf_member_index = IntegerField() + vdu_name = CharField() + scaling_record = ForeignKeyField(ScalingRecord, related_name='scaling_alarms') + + +class DatabaseManager: + def create_tables(self): + try: + db.connect() + db.create_tables([ScalingRecord, ScalingAlarm]) + db.close() + except Exception as e: + log.exception("Error creating tables: ") diff --git a/osm_policy_module/core/singleton.py b/osm_policy_module/core/singleton.py new file mode 100644 index 0000000..9b8db5d --- /dev/null +++ b/osm_policy_module/core/singleton.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +"""Simple singleton class.""" + +from __future__ import unicode_literals + + +class Singleton(object): + """Simple singleton class.""" + + def __init__(self, decorated): + """Initialize singleton instance.""" + self._decorated = decorated + + def instance(self): + """Return singleton instance.""" + try: + return self._instance + except AttributeError: + self._instance = self._decorated() + return self._instance diff --git a/osm_policy_module/tests/__init__.py b/osm_policy_module/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/osm_policy_module/tests/examples/cirros_vdu_scaling_nsd.yaml b/osm_policy_module/tests/examples/cirros_vdu_scaling_nsd.yaml new file mode 100644 index 0000000..a9e0abd --- /dev/null +++ b/osm_policy_module/tests/examples/cirros_vdu_scaling_nsd.yaml @@ -0,0 +1,42 @@ +nsd:nsd-catalog: + nsd: + - id: cirros_vdu_scaling_ns + name: cirros_vdu_scaling_ns + short-name: cirros_vdu_scaling_ns + description: Simple NS example with a cirros_vdu_scaling_vnf + vendor: OSM + version: '1.0' + + # Place the logo as png in icons directory and provide the name here + logo: osm.png + + # Specify the VNFDs that are part of this NSD + constituent-vnfd: + # The member-vnf-index needs to be unique, starting from 1 + # vnfd-id-ref is the id of the VNFD + # Multiple constituent VNFDs can be specified + - member-vnf-index: '1' + vnfd-id-ref: cirros_vdu_scaling_vnf + - member-vnf-index: '2' + vnfd-id-ref: cirros_vdu_scaling_vnf + vld: + # Networks for the VNFs + - id: cirros_nsd_vld1 + name: cirros_nsd_vld1 + type: ELAN + mgmt-network: 'true' + # vim-network-name: + # provider-network: + # segmentation_id: + vnfd-connection-point-ref: + # Specify the constituent VNFs + # member-vnf-index-ref - entry from constituent vnf + # vnfd-id-ref - VNFD id + # vnfd-connection-point-ref - connection point name in the VNFD + - member-vnf-index-ref: 1 + vnfd-id-ref: cirros_vdu_scaling_vnf + # NOTE: Validate the entry below + vnfd-connection-point-ref: eth0 + - member-vnf-index-ref: 2 + vnfd-id-ref: cirros_vdu_scaling_vnf + vnfd-connection-point-ref: eth0 \ No newline at end of file diff --git a/osm_policy_module/tests/examples/cirros_vdu_scaling_vnfd.yaml b/osm_policy_module/tests/examples/cirros_vdu_scaling_vnfd.yaml new file mode 100644 index 0000000..dc599f5 --- /dev/null +++ b/osm_policy_module/tests/examples/cirros_vdu_scaling_vnfd.yaml @@ -0,0 +1,74 @@ +vnfd:vnfd-catalog: + vnfd: + - id: cirros_vdu_scaling_vnf + name: cirros_vdu_scaling_vnf + short-name: cirros_vdu_scaling_vnf + description: Simple VNF example with a cirros and a scaling group descriptor + vendor: OSM + version: '1.0' + # Place the logo as png in icons directory and provide the name here + logo: cirros-64.png + # Management interface + mgmt-interface: + cp: eth0 + # Atleast one VDU need to be specified + vdu: + - id: cirros_vnfd-VM + name: cirros_vnfd-VM + description: cirros_vnfd-VM + count: 1 + + # Flavour of the VM to be instantiated for the VDU + # flavor below can fit into m1.micro + vm-flavor: + vcpu-count: 1 + memory-mb: 256 + storage-gb: 2 + # Image/checksum or image including the full path + image: 'cirros034' + #checksum: + interface: + # Specify the external interfaces + # There can be multiple interfaces defined + - name: eth0 + type: EXTERNAL + virtual-interface: + type: VIRTIO + bandwidth: '0' + vpci: 0000:00:0a.0 + external-connection-point-ref: eth0 + monitoring-param: + - id: "cirros_vnfd-VM_memory_util" + nfvi-metric: "average_memory_utilization" # The associated NFVI metric to be monitored. Id of the metric + #interface-name-ref: reference to interface name, required for some metrics + connection-point: + - name: eth0 + type: VPORT + scaling-group-descriptor: + - name: "scale_cirros_vnfd-VM" + min-instance-count: 1 + max-instance-count: 10 + scaling-policy: + - name: "auto_memory_util_above_threshold" + scaling-type: "automatic" + threshold-time: 10 + cooldown-time: 60 + scaling-criteria: + - name: "group1_memory_util_above_threshold" + scale-in-threshold: 20 + scale-in-relational-operation: "LT" + scale-out-threshold: 80 + scale-out-relational-operation: "GT" + vnf-monitoring-param-ref: "cirros_vnf_memory_util" + vdu: + - vdu-id-ref: cirros_vnfd-VM + count: 1 + # scaling-config-action: # Para utilizar charms + # - trigger: post-scale-out + # vnf-config-primitive-name-ref: + monitoring-param: + - id: "cirros_vnf_memory_util" + name: "cirros_vnf_memory_util" + aggregation-type: AVERAGE + vdu-ref: "cirros_vnfd-VM" + vdu-monitoring-param-ref: "cirros_vnfd-VM_memory_util" \ No newline at end of file diff --git a/osm_policy_module/tests/examples/instantiated.json b/osm_policy_module/tests/examples/instantiated.json new file mode 100644 index 0000000..88b610d --- /dev/null +++ b/osm_policy_module/tests/examples/instantiated.json @@ -0,0 +1,4 @@ +{ + "nsr_id": "test_nsr_id", + "nslcmop_id": "test_nslcmop_id" +} \ No newline at end of file diff --git a/osm_policy_module/tests/integration/__init__.py b/osm_policy_module/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/osm_policy_module/tests/integration/test_kafka_messages.py b/osm_policy_module/tests/integration/test_kafka_messages.py new file mode 100644 index 0000000..d17d2b2 --- /dev/null +++ b/osm_policy_module/tests/integration/test_kafka_messages.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import json +import logging +import os +import sys +import unittest + +from kafka import KafkaProducer, KafkaConsumer +from kafka.errors import KafkaError + +from osm_policy_module.core.agent import PolicyModuleAgent +from osm_policy_module.core.config import Config + +log = logging.getLogger() +log.level = logging.INFO +stream_handler = logging.StreamHandler(sys.stdout) +log.addHandler(stream_handler) + + +class KafkaMessagesTest(unittest.TestCase): + def setUp(self): + try: + cfg = Config.instance() + kafka_server = '{}:{}'.format(cfg.OSMPOL_MESSAGE_HOST, + cfg.OSMPOL_MESSAGE_PORT) + self.producer = KafkaProducer(bootstrap_servers=kafka_server, + key_serializer=str.encode, + value_serializer=str.encode) + self.consumer = KafkaConsumer(bootstrap_servers=kafka_server, + key_deserializer=bytes.decode, + value_deserializer=bytes.decode, + auto_offset_reset='earliest', + consumer_timeout_ms=5000) + self.consumer.subscribe(['ns']) + except KafkaError: + self.skipTest('Kafka server not present.') + + def tearDown(self): + self.producer.close() + self.consumer.close() + + def test_send_instantiated_msg(self): + with open( + os.path.join(os.path.dirname(__file__), '../examples/instantiated.json')) as file: + payload = json.load(file) + self.producer.send('ns', json.dumps(payload), key="instantiated") + self.producer.flush() + + for message in self.consumer: + if message.key == 'instantiated': + self.assertIsNotNone(message.value) + return + self.fail("No message received in consumer") + + +if __name__ == '__main__': + unittest.main() diff --git a/osm_policy_module/tests/integration/test_policy_agent.py b/osm_policy_module/tests/integration/test_policy_agent.py new file mode 100644 index 0000000..4de0dbb --- /dev/null +++ b/osm_policy_module/tests/integration/test_policy_agent.py @@ -0,0 +1,456 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import logging +import sys +import unittest +import uuid +from unittest.mock import patch, Mock + +from kafka import KafkaProducer +from osm_common.dbmongo import DbMongo +from osm_policy_module.core import database +from peewee import SqliteDatabase + +from osm_policy_module.common.mon_client import MonClient +from osm_policy_module.core.agent import PolicyModuleAgent +from osm_policy_module.core.database import ScalingRecord, ScalingAlarm, BaseModel + +log = logging.getLogger() +log.level = logging.INFO +stream_handler = logging.StreamHandler(sys.stdout) +log.addHandler(stream_handler) + +nsr_record_mock = { + "_id": "87776f33-b67c-417a-8119-cb08e4098951", + "crete-time": 1535392482.0044956, + "operational-status": "running", + "ssh-authorized-key": None, + "name-ref": "cirros_ns", + "nsd": { + "name": "cirros_vdu_scaling_ns", + "_id": "d7c8bd3c-eb39-4514-8847-19f01345524f", + "vld": [ + { + "id": "cirros_nsd_vld1", + "name": "cirros_nsd_vld1", + "type": "ELAN", + "mgmt-network": "true", + "vnfd-connection-point-ref": [ + { + "vnfd-id-ref": "cirros_vdu_scaling_vnf", + "member-vnf-index-ref": 1, + "vnfd-connection-point-ref": "eth0" + }, + { + "vnfd-id-ref": "cirros_vdu_scaling_vnf", + "member-vnf-index-ref": 2, + "vnfd-connection-point-ref": "eth0" + } + ] + } + ], + "vendor": "OSM", + "constituent-vnfd": [ + { + "member-vnf-index": "1", + "vnfd-id-ref": "cirros_vdu_scaling_vnf" + }, + { + "member-vnf-index": "2", + "vnfd-id-ref": "cirros_vdu_scaling_vnf" + } + ], + "version": "1.0", + "id": "cirros_vdu_scaling_ns", + "description": "Simple NS example with a cirros_vdu_scaling_vnf", + "logo": "osm.png", + "_admin": { + "created": 1535392246.499733, + "userDefinedData": { + + }, + "usageSate": "NOT_IN_USE", + "storage": { + "zipfile": "package.tar.gz", + "fs": "local", + "path": "/app/storage/", + "folder": "d7c8bd3c-eb39-4514-8847-19f01345524f", + "pkg-dir": "cirros_nsd", + "descriptor": "cirros_nsd/cirros_vdu_scaling_nsd.yaml" + }, + "onboardingState": "ONBOARDED", + "modified": 1535392246.499733, + "projects_read": [ + "admin" + ], + "operationalState": "ENABLED", + "projects_write": [ + "admin" + ] + }, + "short-name": "cirros_vdu_scaling_ns" + }, + "id": "87776f33-b67c-417a-8119-cb08e4098951", + "config-status": "configured", + "operational-events": [], + "_admin": { + "created": 1535392482.0084584, + "projects_read": [ + "admin" + ], + "nsState": "INSTANTIATED", + "modified": 1535392482.0084584, + "projects_write": [ + "admin" + ], + "deployed": { + "RO": { + "vnfd_id": { + "cirros_vdu_scaling_vnf": "7445e347-fe2f-431a-abc2-8b9be3d093c6" + }, + "nsd_id": "92c56cf0-f8fa-488c-9afb-9f3d78ae6bbb", + "nsr_id": "637e12cd-c201-4c44-8ebd-70fb57a4dcee", + "nsr_status": "BUILD" + } + } + }, + "nsd-ref": "cirros_vdu_scaling_ns", + "name": "cirros_ns", + "resource-orchestrator": "osmopenmano", + "instantiate_params": { + "nsDescription": "default description", + "nsdId": "d7c8bd3c-eb39-4514-8847-19f01345524f", + "nsr_id": "87776f33-b67c-417a-8119-cb08e4098951", + "nsName": "cirros_ns", + "vimAccountId": "be48ae31-1d46-4892-a4b4-d69abd55714b" + }, + "description": "default description", + "constituent-vnfr-ref": [ + "0d9d06ad-3fc2-418c-9934-465e815fafe2", + "3336eb44-77df-4c4f-9881-d2828d259864" + ], + "admin-status": "ENABLED", + "detailed-status": "done", + "datacenter": "be48ae31-1d46-4892-a4b4-d69abd55714b", + "orchestration-progress": { + + }, + "short-name": "cirros_ns", + "ns-instance-config-ref": "87776f33-b67c-417a-8119-cb08e4098951", + "nsd-name-ref": "cirros_vdu_scaling_ns", + "admin": { + "deployed": { + "RO": { + "nsr_status": "ACTIVE" + } + } + } +} + +vnfr_record_mocks = [ + { + "_id": "0d9d06ad-3fc2-418c-9934-465e815fafe2", + "ip-address": "192.168.160.2", + "created-time": 1535392482.0044956, + "vim-account-id": "be48ae31-1d46-4892-a4b4-d69abd55714b", + "vdur": [ + { + "interfaces": [ + { + "mac-address": "fa:16:3e:71:fd:b8", + "name": "eth0", + "ip-address": "192.168.160.2" + } + ], + "status": "ACTIVE", + "vim-id": "63a65636-9fc8-4022-b070-980823e6266a", + "name": "cirros_ns-1-cirros_vnfd-VM-1", + "status-detailed": None, + "ip-address": "192.168.160.2", + "vdu-id-ref": "cirros_vnfd-VM" + } + ], + "id": "0d9d06ad-3fc2-418c-9934-465e815fafe2", + "vnfd-ref": "cirros_vdu_scaling_vnf", + "vnfd-id": "63f44c41-45ee-456b-b10d-5f08fb1796e0", + "_admin": { + "created": 1535392482.0067868, + "projects_read": [ + "admin" + ], + "modified": 1535392482.0067868, + "projects_write": [ + "admin" + ] + }, + "nsr-id-ref": "87776f33-b67c-417a-8119-cb08e4098951", + "member-vnf-index-ref": "1", + "connection-point": [ + { + "name": "eth0", + "id": None, + "connection-point-id": None + } + ] + }, + { + "_id": "3336eb44-77df-4c4f-9881-d2828d259864", + "ip-address": "192.168.160.10", + "created-time": 1535392482.0044956, + "vim-account-id": "be48ae31-1d46-4892-a4b4-d69abd55714b", + "vdur": [ + { + "interfaces": [ + { + "mac-address": "fa:16:3e:1e:76:e8", + "name": "eth0", + "ip-address": "192.168.160.10" + } + ], + "status": "ACTIVE", + "vim-id": "a154b8d3-2b10-421a-a51d-4b391d9bd366", + "name": "cirros_ns-2-cirros_vnfd-VM-1", + "status-detailed": None, + "ip-address": "192.168.160.10", + "vdu-id-ref": "cirros_vnfd-VM" + } + ], + "id": "3336eb44-77df-4c4f-9881-d2828d259864", + "vnfd-ref": "cirros_vdu_scaling_vnf", + "vnfd-id": "63f44c41-45ee-456b-b10d-5f08fb1796e0", + "_admin": { + "created": 1535392482.0076294, + "projects_read": [ + "admin" + ], + "modified": 1535392482.0076294, + "projects_write": [ + "admin" + ] + }, + "nsr-id-ref": "87776f33-b67c-417a-8119-cb08e4098951", + "member-vnf-index-ref": "2", + "connection-point": [ + { + "name": "eth0", + "id": None, + "connection-point-id": None + } + ]}] + +nsd_record_mock = {'name': 'cirros_vdu_scaling_ns', + 'version': '1.0', + 'short-name': 'cirros_vdu_scaling_ns', + 'logo': 'osm.png', + 'id': 'cirros_vdu_scaling_ns', + 'description': 'Simple NS example with a cirros_vdu_scaling_vnf', + 'vendor': 'OSM', + 'vld': [ + {'name': 'cirros_nsd_vld1', + 'type': 'ELAN', + 'id': 'cirros_nsd_vld1', + 'mgmt-network': 'true', + 'vnfd-connection-point-ref': [ + {'vnfd-id-ref': 'cirros_vdu_scaling_vnf', + 'vnfd-connection-point-ref': 'eth0', + 'member-vnf-index-ref': 1}, + {'vnfd-id-ref': 'cirros_vdu_scaling_vnf', + 'vnfd-connection-point-ref': 'eth0', + 'member-vnf-index-ref': 2}]}], + 'constituent-vnfd': [{'vnfd-id-ref': 'cirros_vdu_scaling_vnf', + 'member-vnf-index': '1'}, + {'vnfd-id-ref': 'cirros_vdu_scaling_vnf', + 'member-vnf-index': '2'}]} + +vnfd_record_mock = { + "_id": "63f44c41-45ee-456b-b10d-5f08fb1796e0", + "name": "cirros_vdu_scaling_vnf", + "vendor": "OSM", + "vdu": [ + { + "name": "cirros_vnfd-VM", + "monitoring-param": [ + { + "id": "cirros_vnfd-VM_memory_util", + "nfvi-metric": "average_memory_utilization" + } + ], + "vm-flavor": { + "vcpu-count": 1, + "memory-mb": 256, + "storage-gb": 2 + }, + "description": "cirros_vnfd-VM", + "count": 1, + "id": "cirros_vnfd-VM", + "interface": [ + { + "name": "eth0", + "external-connection-point-ref": "eth0", + "type": "EXTERNAL", + "virtual-interface": { + "bandwidth": "0", + "type": "VIRTIO", + "vpci": "0000:00:0a.0" + } + } + ], + "image": "cirros034" + } + ], + "monitoring-param": [ + { + "id": "cirros_vnf_memory_util", + "name": "cirros_vnf_memory_util", + "aggregation-type": "AVERAGE", + "vdu-monitoring-param-ref": "cirros_vnfd-VM_memory_util", + "vdu-ref": "cirros_vnfd-VM" + } + ], + "description": "Simple VNF example with a cirros and a scaling group descriptor", + "id": "cirros_vdu_scaling_vnf", + "logo": "cirros-64.png", + "version": "1.0", + "connection-point": [ + { + "name": "eth0", + "type": "VPORT" + } + ], + "mgmt-interface": { + "cp": "eth0" + }, + "scaling-group-descriptor": [ + { + "name": "scale_cirros_vnfd-VM", + "min-instance-count": 1, + "vdu": [ + { + "count": 1, + "vdu-id-ref": "cirros_vnfd-VM" + } + ], + "max-instance-count": 10, + "scaling-policy": [ + { + "name": "auto_memory_util_above_threshold", + "scaling-type": "automatic", + "cooldown-time": 60, + "threshold-time": 10, + "scaling-criteria": [ + { + "name": "group1_memory_util_above_threshold", + "vnf-monitoring-param-ref": "cirros_vnf_memory_util", + "scale-out-threshold": 80, + "scale-out-relational-operation": "GT", + "scale-in-relational-operation": "LT", + "scale-in-threshold": 20 + } + ] + } + ] + } + ], + "short-name": "cirros_vdu_scaling_vnf", + "_admin": { + "created": 1535392242.6281035, + "modified": 1535392242.6281035, + "storage": { + "zipfile": "package.tar.gz", + "pkg-dir": "cirros_vnf", + "path": "/app/storage/", + "folder": "63f44c41-45ee-456b-b10d-5f08fb1796e0", + "fs": "local", + "descriptor": "cirros_vnf/cirros_vdu_scaling_vnfd.yaml" + }, + "usageSate": "NOT_IN_USE", + "onboardingState": "ONBOARDED", + "userDefinedData": { + + }, + "projects_read": [ + "admin" + ], + "operationalState": "ENABLED", + "projects_write": [ + "admin" + ] + } +} + +test_db = SqliteDatabase(':memory:') + +MODELS = [ScalingRecord, ScalingAlarm] + + +class PolicyModuleAgentTest(unittest.TestCase): + def setUp(self): + super() + database.db = test_db + test_db.bind(MODELS) + test_db.connect() + test_db.drop_tables(MODELS) + test_db.create_tables(MODELS) + + def tearDown(self): + super() + + @patch.object(DbMongo, 'db_connect', Mock()) + @patch.object(KafkaProducer, '__init__') + @patch.object(MonClient, 'create_alarm') + @patch.object(PolicyModuleAgent, '_get_vnfd') + @patch.object(PolicyModuleAgent, '_get_nsr') + @patch.object(PolicyModuleAgent, '_get_vnfr') + def test_configure_scaling_groups(self, get_vnfr, get_nsr, get_vnfd, create_alarm, kafka_producer_init): + def _test_configure_scaling_groups_get_vnfr(*args, **kwargs): + if '1' in args[1]: + return vnfr_record_mocks[0] + if '2' in args[1]: + return vnfr_record_mocks[1] + + def _test_configure_scaling_groups_create_alarm(*args, **kwargs): + return uuid.uuid4() + + kafka_producer_init.return_value = None + get_vnfr.side_effect = _test_configure_scaling_groups_get_vnfr + get_nsr.return_value = nsr_record_mock + get_vnfd.return_value = vnfd_record_mock + create_alarm.side_effect = _test_configure_scaling_groups_create_alarm + agent = PolicyModuleAgent() + agent._configure_scaling_groups("test_nsr_id") + create_alarm.assert_any_call(metric_name='average_memory_utilization', + ns_id='test_nsr_id', + operation='GT', + statistic='AVERAGE', + threshold=80, + vdu_name='cirros_vnfd-VM', + vnf_member_index='1') + scaling_record = ScalingRecord.get() + self.assertEqual(scaling_record.name, 'scale_cirros_vnfd-VM') + self.assertEqual(scaling_record.nsr_id, 'test_nsr_id') + self.assertIsNotNone(scaling_record) + + +if __name__ == '__main__': + unittest.main() diff --git a/osm_policy_module/tests/unit/__init__.py b/osm_policy_module/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/osm_policy_module/tests/unit/test_policy_agent.py b/osm_policy_module/tests/unit/test_policy_agent.py new file mode 100644 index 0000000..5e59901 --- /dev/null +++ b/osm_policy_module/tests/unit/test_policy_agent.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import unittest + +from osm_policy_module.core.agent import PolicyModuleAgent + + +class PolicyAgentTest(unittest.TestCase): + def setUp(self): + self.agent = PolicyModuleAgent() + + +if __name__ == '__main__': + unittest.main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cded148 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +kafka==1.3.* +peewee==3.1.* +jsonschema==2.6.* +six==1.11.* +pyyaml==3.* +git+https://osm.etsi.org/gerrit/osm/common.git#egg=osm-common diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c751cbf --- /dev/null +++ b/setup.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +from setuptools import setup + +_name = 'osm_policy_module' +_version_command = ('git describe --match v* --tags --long --dirty', 'pep440-git') +_author = "Benjamín Díaz" +_author_email = 'bdiaz@whitestack.com' +_description = 'OSM Policy Module' +_maintainer = 'Benjamín Díaz' +_maintainer_email = 'bdiaz@whitestack.com' +_license = 'Apache 2.0' +_url = 'https://osm.etsi.org/gitweb/?p=osm/MON.git;a=tree' + +setup( + name=_name, + version_command=_version_command, + description=_description, + long_description=open('README.rst').read(), + author=_author, + author_email=_author_email, + maintainer=_maintainer, + maintainer_email=_maintainer_email, + url=_url, + license=_license, + packages=setuptools.find_packages(), + include_package_data=True, + install_requires=[ + "kafka==1.3.*", + "peewee==3.1.*", + "jsonschema==2.6.*", + "six==1.11.*", + "pyyaml==3.*", + "osm-common" + ], + entry_points={ + "console_scripts": [ + "osm-policy-agent = osm_policy_module.cmd.policy_module_agent:main", + ] + }, + dependency_links=[ + 'git+https://osm.etsi.org/gerrit/osm/common.git#egg=osm-common' + ], + setup_requires=['setuptools-version-command'] +) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..b404738 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,23 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: helena.mcgough@intel.com or adrian.hoban@intel.com +## +flake8<3.0 +mock diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..ca49be9 --- /dev/null +++ b/tox.ini @@ -0,0 +1,57 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: helena.mcgough@intel.com or adrian.hoban@intel.com +## + +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. +[tox] +envlist = py3, flake8 +toxworkdir={homedir}/.tox + +[testenv] +basepythons = python3 +commands=python3 -m unittest discover -v +install_command = python3 -m pip install -r requirements.txt -U {opts} {packages} +deps = -r{toxinidir}/test-requirements.txt + +[testenv:flake8] +basepython = python3 +deps = flake8 +commands = + flake8 osm_policy_module + +[testenv:build] +basepython = python3 +deps = stdeb + setuptools-version-command +commands = python3 setup.py --command-packages=stdeb.command bdist_deb + +[flake8] +# E123, E125 skipped as they are invalid PEP-8. +max-line-length = 120 +show-source = True +ignore = E123,E125,E241 +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,devops_stages/*,.rst + + -- 2.25.1