osm_policy_module/tests/integration/test_policy_agent.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Whitestack, LLC
   4 # *************************************************************
   5
   6 # This file is part of OSM Monitoring module
   7 # All Rights Reserved to Whitestack, LLC
   8
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  23 ##
  24 import asyncio
  25 import logging
  26 import os
  27 import sys
  28 import unittest
  29 import uuid
  30 from unittest.mock import patch, Mock
  31
  32 from kafka import KafkaProducer
  33 from osm_common.dbmongo import DbMongo
  34 from playhouse.db_url import connect
  35
  36 from osm_policy_module.common.common_db_client import CommonDbClient
  37 from osm_policy_module.common.mon_client import MonClient
  38 from osm_policy_module.core import database
  39 from osm_policy_module.core.agent import PolicyModuleAgent
  40 from osm_policy_module.core.config import Config
  41 from osm_policy_module.core.database import ScalingGroup, ScalingAlarm, ScalingPolicy, ScalingCriteria, VnfAlarm, \
  42     AlarmAction
  43
  44 log = logging.getLogger()
  45 log.level = logging.INFO
  46 stream_handler = logging.StreamHandler(sys.stdout)
  47 log.addHandler(stream_handler)
  48
  49 nsr_record_mock = {
  50     "_id": "87776f33-b67c-417a-8119-cb08e4098951",
  51     "crete-time": 1535392482.0044956,
  52     "operational-status": "running",
  53     "ssh-authorized-key": None,
  54     "name-ref": "cirros_ns",
  55     "nsd": {
  56         "name": "cirros_vdu_scaling_ns",
  57         "_id": "d7c8bd3c-eb39-4514-8847-19f01345524f",
  58         "vld": [
  59             {
  60                 "id": "cirros_nsd_vld1",
  61                 "name": "cirros_nsd_vld1",
  62                 "type": "ELAN",
  63                 "mgmt-network": "true",
  64                 "vnfd-connection-point-ref": [
  65                     {
  66                         "vnfd-id-ref": "cirros_vdu_scaling_vnf",
  67                         "member-vnf-index-ref": 1,
  68                         "vnfd-connection-point-ref": "eth0"
  69                     },
  70                     {
  71                         "vnfd-id-ref": "cirros_vdu_scaling_vnf",
  72                         "member-vnf-index-ref": 2,
  73                         "vnfd-connection-point-ref": "eth0"
  74                     }
  75                 ]
  76             }
  77         ],
  78         "vendor": "OSM",
  79         "constituent-vnfd": [
  80             {
  81                 "member-vnf-index": "1",
  82                 "vnfd-id-ref": "cirros_vdu_scaling_vnf"
  83             },
  84             {
  85                 "member-vnf-index": "2",
  86                 "vnfd-id-ref": "cirros_vdu_scaling_vnf"
  87             }
  88         ],
  89         "version": "1.0",
  90         "id": "cirros_vdu_scaling_ns",
  91         "description": "Simple NS example with a cirros_vdu_scaling_vnf",
  92         "logo": "osm.png",
  93         "_admin": {
  94             "created": 1535392246.499733,
  95             "userDefinedData": {
  96
  97             },
  98             "usageSate": "NOT_IN_USE",
  99             "storage": {
 100                 "zipfile": "package.tar.gz",
 101                 "fs": "local",
 102                 "path": "/app/storage/",
 103                 "folder": "d7c8bd3c-eb39-4514-8847-19f01345524f",
 104                 "pkg-dir": "cirros_nsd",
 105                 "descriptor": "cirros_nsd/cirros_vdu_scaling_nsd.yaml"
 106             },
 107             "onboardingState": "ONBOARDED",
 108             "modified": 1535392246.499733,
 109             "projects_read": [
 110                 "admin"
 111             ],
 112             "operationalState": "ENABLED",
 113             "projects_write": [
 114                 "admin"
 115             ]
 116         },
 117         "short-name": "cirros_vdu_scaling_ns"
 118     },
 119     "id": "87776f33-b67c-417a-8119-cb08e4098951",
 120     "config-status": "configured",
 121     "operational-events": [],
 122     "_admin": {
 123         "created": 1535392482.0084584,
 124         "projects_read": [
 125             "admin"
 126         ],
 127         "nsState": "INSTANTIATED",
 128         "modified": 1535392482.0084584,
 129         "projects_write": [
 130             "admin"
 131         ],
 132         "deployed": {
 133             "RO": {
 134                 "vnfd_id": {
 135                     "cirros_vdu_scaling_vnf": "7445e347-fe2f-431a-abc2-8b9be3d093c6"
 136                 },
 137                 "nsd_id": "92c56cf0-f8fa-488c-9afb-9f3d78ae6bbb",
 138                 "nsr_id": "637e12cd-c201-4c44-8ebd-70fb57a4dcee",
 139                 "nsr_status": "BUILD"
 140             }
 141         }
 142     },
 143     "nsd-ref": "cirros_vdu_scaling_ns",
 144     "name": "cirros_ns",
 145     "resource-orchestrator": "osmopenmano",
 146     "instantiate_params": {
 147         "nsDescription": "default description",
 148         "nsdId": "d7c8bd3c-eb39-4514-8847-19f01345524f",
 149         "nsr_id": "87776f33-b67c-417a-8119-cb08e4098951",
 150         "nsName": "cirros_ns",
 151         "vimAccountId": "be48ae31-1d46-4892-a4b4-d69abd55714b"
 152     },
 153     "description": "default description",
 154     "constituent-vnfr-ref": [
 155         "0d9d06ad-3fc2-418c-9934-465e815fafe2",
 156         "3336eb44-77df-4c4f-9881-d2828d259864"
 157     ],
 158     "admin-status": "ENABLED",
 159     "detailed-status": "done",
 160     "datacenter": "be48ae31-1d46-4892-a4b4-d69abd55714b",
 161     "orchestration-progress": {
 162
 163     },
 164     "short-name": "cirros_ns",
 165     "ns-instance-config-ref": "87776f33-b67c-417a-8119-cb08e4098951",
 166     "nsd-name-ref": "cirros_vdu_scaling_ns",
 167     "admin": {
 168         "deployed": {
 169             "RO": {
 170                 "nsr_status": "ACTIVE"
 171             }
 172         }
 173     }
 174 }
 175
 176 vnfr_record_mocks = [
 177     {
 178         "_id": "0d9d06ad-3fc2-418c-9934-465e815fafe2",
 179         "ip-address": "192.168.160.2",
 180         "created-time": 1535392482.0044956,
 181         "vim-account-id": "be48ae31-1d46-4892-a4b4-d69abd55714b",
 182         "vdur": [
 183             {
 184                 "interfaces": [
 185                     {
 186                         "mac-address": "fa:16:3e:71:fd:b8",
 187                         "name": "eth0",
 188                         "ip-address": "192.168.160.2"
 189                     }
 190                 ],
 191                 "status": "ACTIVE",
 192                 "vim-id": "63a65636-9fc8-4022-b070-980823e6266a",
 193                 "name": "cirros_ns-1-cirros_vnfd-VM-1",
 194                 "status-detailed": None,
 195                 "ip-address": "192.168.160.2",
 196                 "vdu-id-ref": "cirros_vnfd-VM"
 197             }
 198         ],
 199         "id": "0d9d06ad-3fc2-418c-9934-465e815fafe2",
 200         "vnfd-ref": "cirros_vdu_scaling_vnf",
 201         "vnfd-id": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 202         "_admin": {
 203             "created": 1535392482.0067868,
 204             "projects_read": [
 205                 "admin"
 206             ],
 207             "modified": 1535392482.0067868,
 208             "projects_write": [
 209                 "admin"
 210             ]
 211         },
 212         "nsr-id-ref": "87776f33-b67c-417a-8119-cb08e4098951",
 213         "member-vnf-index-ref": "1",
 214         "connection-point": [
 215             {
 216                 "name": "eth0",
 217                 "id": None,
 218                 "connection-point-id": None
 219             }
 220         ]
 221     },
 222     {
 223         "_id": "3336eb44-77df-4c4f-9881-d2828d259864",
 224         "ip-address": "192.168.160.10",
 225         "created-time": 1535392482.0044956,
 226         "vim-account-id": "be48ae31-1d46-4892-a4b4-d69abd55714b",
 227         "vdur": [
 228             {
 229                 "interfaces": [
 230                     {
 231                         "mac-address": "fa:16:3e:1e:76:e8",
 232                         "name": "eth0",
 233                         "ip-address": "192.168.160.10"
 234                     }
 235                 ],
 236                 "status": "ACTIVE",
 237                 "vim-id": "a154b8d3-2b10-421a-a51d-4b391d9bd366",
 238                 "name": "cirros_ns-2-cirros_vnfd-VM-1",
 239                 "status-detailed": None,
 240                 "ip-address": "192.168.160.10",
 241                 "vdu-id-ref": "cirros_vnfd-VM"
 242             }
 243         ],
 244         "id": "3336eb44-77df-4c4f-9881-d2828d259864",
 245         "vnfd-ref": "cirros_vdu_scaling_vnf",
 246         "vnfd-id": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 247         "_admin": {
 248             "created": 1535392482.0076294,
 249             "projects_read": [
 250                 "admin"
 251             ],
 252             "modified": 1535392482.0076294,
 253             "projects_write": [
 254                 "admin"
 255             ]
 256         },
 257         "nsr-id-ref": "87776f33-b67c-417a-8119-cb08e4098951",
 258         "member-vnf-index-ref": "2",
 259         "connection-point": [
 260             {
 261                 "name": "eth0",
 262                 "id": None,
 263                 "connection-point-id": None
 264             }
 265         ]}]
 266
 267 nsd_record_mock = {'name': 'cirros_vdu_scaling_ns',
 268                    'version': '1.0',
 269                    'short-name': 'cirros_vdu_scaling_ns',
 270                    'logo': 'osm.png',
 271                    'id': 'cirros_vdu_scaling_ns',
 272                    'description': 'Simple NS example with a cirros_vdu_scaling_vnf',
 273                    'vendor': 'OSM',
 274                    'vld': [
 275                        {'name': 'cirros_nsd_vld1',
 276                         'type': 'ELAN',
 277                         'id': 'cirros_nsd_vld1',
 278                         'mgmt-network': 'true',
 279                         'vnfd-connection-point-ref': [
 280                             {'vnfd-id-ref': 'cirros_vdu_scaling_vnf',
 281                              'vnfd-connection-point-ref': 'eth0',
 282                              'member-vnf-index-ref': 1},
 283                             {'vnfd-id-ref': 'cirros_vdu_scaling_vnf',
 284                              'vnfd-connection-point-ref': 'eth0',
 285                              'member-vnf-index-ref': 2}]}],
 286                    'constituent-vnfd': [{'vnfd-id-ref': 'cirros_vdu_scaling_vnf',
 287                                          'member-vnf-index': '1'},
 288                                         {'vnfd-id-ref': 'cirros_vdu_scaling_vnf',
 289                                          'member-vnf-index': '2'}]}
 290
 291 vnfd_record_mock = {
 292     "_id": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 293     "name": "cirros_vdu_scaling_vnf",
 294     "vendor": "OSM",
 295     "vdu": [
 296         {
 297             "name": "cirros_vnfd-VM",
 298             "monitoring-param": [
 299                 {
 300                     "id": "cirros_vnfd-VM_memory_util",
 301                     "nfvi-metric": "average_memory_utilization"
 302                 }
 303             ],
 304             "vm-flavor": {
 305                 "vcpu-count": 1,
 306                 "memory-mb": 256,
 307                 "storage-gb": 2
 308             },
 309             "description": "cirros_vnfd-VM",
 310             "count": 1,
 311             "id": "cirros_vnfd-VM",
 312             "interface": [
 313                 {
 314                     "name": "eth0",
 315                     "external-connection-point-ref": "eth0",
 316                     "type": "EXTERNAL",
 317                     "virtual-interface": {
 318                         "bandwidth": "0",
 319                         "type": "VIRTIO",
 320                         "vpci": "0000:00:0a.0"
 321                     }
 322                 }
 323             ],
 324             "image": "cirros034",
 325             "alarm": [
 326                 {
 327                     "value": 20.0000,
 328                     "actions": {
 329                         "insufficient-data": [
 330                             {
 331                                 "url": "localhost:9090"
 332                             }
 333                         ],
 334                         "ok": [
 335                             {
 336                                 "url": "localhost:9090"
 337                             }
 338                         ],
 339                         "alarm": [
 340                             {
 341                                 "url": "localhost:9090"
 342                             }
 343                         ]
 344                     },
 345                     "alarm-id": "alarm-1",
 346                     "operation": "LT",
 347                     "vnf-monitoring-param-ref": "cirros_vnf_memory_util"
 348                 }
 349             ]
 350         }
 351     ],
 352     "monitoring-param": [
 353         {
 354             "id": "cirros_vnf_memory_util",
 355             "name": "cirros_vnf_memory_util",
 356             "aggregation-type": "AVERAGE",
 357             "vdu-monitoring-param": {
 358                 "vdu-monitoring-param-ref": "cirros_vnfd-VM_memory_util",
 359                 "vdu-ref": "cirros_vnfd-VM"
 360             }
 361         },
 362         {
 363             "id": "haproxy_users",
 364             "name": "haproxy_users",
 365             "aggregation-type": "AVERAGE",
 366             "vnf-metric":
 367                 {"vnf-metric-name-ref": "users"}
 368         }
 369     ],
 370     "vdu-configuration": {
 371         "juju": {
 372             "charm": "testmetrics",
 373             "proxy": True
 374         },
 375         "metrics": {
 376             "name": "users"
 377         }
 378     },
 379     "description": "Simple VNF example with a cirros and a scaling group descriptor",
 380     "id": "cirros_vdu_scaling_vnf",
 381     "logo": "cirros-64.png",
 382     "version": "1.0",
 383     "connection-point": [
 384         {
 385             "name": "eth0",
 386             "type": "VPORT"
 387         }
 388     ],
 389     "mgmt-interface": {
 390         "cp": "eth0"
 391     },
 392     "scaling-group-descriptor": [
 393         {
 394             "name": "scale_cirros_vnfd-VM",
 395             "min-instance-count": 1,
 396             "vdu": [
 397                 {
 398                     "count": 1,
 399                     "vdu-id-ref": "cirros_vnfd-VM"
 400                 }
 401             ],
 402             "max-instance-count": 10,
 403             "scaling-policy": [
 404                 {
 405                     "name": "auto_memory_util_above_threshold",
 406                     "scaling-type": "automatic",
 407                     "cooldown-time": 60,
 408                     "threshold-time": 10,
 409                     "scaling-criteria": [
 410                         {
 411                             "name": "group1_memory_util_above_threshold",
 412                             "vnf-monitoring-param-ref": "cirros_vnf_memory_util",
 413                             "scale-out-threshold": 80,
 414                             "scale-out-relational-operation": "GT",
 415                             "scale-in-relational-operation": "LT",
 416                             "scale-in-threshold": 20
 417                         }
 418                     ]
 419                 }
 420             ]
 421         }
 422     ],
 423     "short-name": "cirros_vdu_scaling_vnf",
 424     "_admin": {
 425         "created": 1535392242.6281035,
 426         "modified": 1535392242.6281035,
 427         "storage": {
 428             "zipfile": "package.tar.gz",
 429             "pkg-dir": "cirros_vnf",
 430             "path": "/app/storage/",
 431             "folder": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 432             "fs": "local",
 433             "descriptor": "cirros_vnf/cirros_vdu_scaling_vnfd.yaml"
 434         },
 435         "usageSate": "NOT_IN_USE",
 436         "onboardingState": "ONBOARDED",
 437         "userDefinedData": {
 438
 439         },
 440         "projects_read": [
 441             "admin"
 442         ],
 443         "operationalState": "ENABLED",
 444         "projects_write": [
 445             "admin"
 446         ]
 447     }
 448 }
 449
 450 MODELS = [ScalingGroup, ScalingPolicy, ScalingCriteria, ScalingAlarm, VnfAlarm, AlarmAction]
 451
 452
 453 class PolicyModuleAgentTest(unittest.TestCase):
 454     def setUp(self):
 455         super()
 456         database.db.initialize(connect('sqlite:///test_db.sqlite'))
 457         database.db.bind(MODELS)
 458         database.db.connect()
 459         database.db.drop_tables(MODELS)
 460         database.db.create_tables(MODELS)
 461         database.db.close()
 462         self.loop = asyncio.new_event_loop()
 463
 464     def tearDown(self):
 465         super()
 466         os.remove('test_db.sqlite')
 467
 468     @patch.object(DbMongo, 'db_connect', Mock())
 469     @patch.object(KafkaProducer, '__init__')
 470     @patch.object(MonClient, 'create_alarm')
 471     @patch.object(CommonDbClient, 'get_vnfd')
 472     @patch.object(CommonDbClient, 'get_nsr')
 473     @patch.object(CommonDbClient, 'get_vnfr')
 474     def test_configure_scaling_groups(self, get_vnfr, get_nsr, get_vnfd, create_alarm, kafka_producer_init):
 475         def _test_configure_scaling_groups_get_vnfr(*args, **kwargs):
 476             if '1' in args[1]:
 477                 return vnfr_record_mocks[0]
 478             if '2' in args[1]:
 479                 return vnfr_record_mocks[1]
 480
 481         async def _test_configure_scaling_groups_create_alarm(*args, **kwargs):
 482             return uuid.uuid4()
 483
 484         kafka_producer_init.return_value = None
 485         get_vnfr.side_effect = _test_configure_scaling_groups_get_vnfr
 486         get_nsr.return_value = nsr_record_mock
 487         get_vnfd.return_value = vnfd_record_mock
 488         create_alarm.side_effect = _test_configure_scaling_groups_create_alarm
 489         config = Config()
 490         agent = PolicyModuleAgent(config, self.loop)
 491         self.loop.run_until_complete(agent.autoscaling_service.configure_scaling_groups("test_nsr_id"))
 492         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 493                                      ns_id='test_nsr_id',
 494                                      operation='GT',
 495                                      statistic='AVERAGE',
 496                                      threshold=80,
 497                                      vdu_name='cirros_ns-1-cirros_vnfd-VM-1',
 498                                      vnf_member_index='1')
 499         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 500                                      ns_id='test_nsr_id',
 501                                      operation='LT',
 502                                      statistic='AVERAGE',
 503                                      threshold=20,
 504                                      vdu_name='cirros_ns-1-cirros_vnfd-VM-1',
 505                                      vnf_member_index='1')
 506         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 507                                      ns_id='test_nsr_id',
 508                                      operation='GT',
 509                                      statistic='AVERAGE',
 510                                      threshold=80,
 511                                      vdu_name='cirros_ns-2-cirros_vnfd-VM-1',
 512                                      vnf_member_index='2')
 513         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 514                                      ns_id='test_nsr_id',
 515                                      operation='LT',
 516                                      statistic='AVERAGE',
 517                                      threshold=20,
 518                                      vdu_name='cirros_ns-2-cirros_vnfd-VM-1',
 519                                      vnf_member_index='2')
 520         scaling_record = ScalingGroup.get()
 521         self.assertEqual(scaling_record.name, 'scale_cirros_vnfd-VM')
 522         self.assertEqual(scaling_record.nsr_id, 'test_nsr_id')
 523
 524     @patch.object(DbMongo, 'db_connect', Mock())
 525     @patch.object(KafkaProducer, '__init__')
 526     @patch.object(MonClient, 'create_alarm')
 527     @patch.object(CommonDbClient, 'get_vnfd')
 528     @patch.object(CommonDbClient, 'get_nsr')
 529     @patch.object(CommonDbClient, 'get_vnfr')
 530     def test_configure_vnf_alarms(self, get_vnfr, get_nsr, get_vnfd, create_alarm, kafka_producer_init):
 531         def _test_configure_scaling_groups_get_vnfr(*args, **kwargs):
 532             if '1' in args[1]:
 533                 return vnfr_record_mocks[0]
 534             if '2' in args[1]:
 535                 return vnfr_record_mocks[1]
 536
 537         async def _test_configure_vnf_alarms_create_alarm(*args, **kwargs):
 538             return uuid.uuid4()
 539
 540         kafka_producer_init.return_value = None
 541         get_vnfr.side_effect = _test_configure_scaling_groups_get_vnfr
 542         get_nsr.return_value = nsr_record_mock
 543         get_vnfd.return_value = vnfd_record_mock
 544         create_alarm.side_effect = _test_configure_vnf_alarms_create_alarm
 545         config = Config()
 546         agent = PolicyModuleAgent(config, self.loop)
 547         self.loop.run_until_complete(agent.alarming_service.configure_vnf_alarms("test_nsr_id"))
 548         create_alarm.assert_any_call(metric_name='cirros_vnf_memory_util',
 549                                      ns_id='test_nsr_id',
 550                                      operation='LT',
 551                                      statistic='AVERAGE',
 552                                      threshold=20.0,
 553                                      vdu_name='cirros_ns-1-cirros_vnfd-VM-1',
 554                                      vnf_member_index='1')
 555         create_alarm.assert_any_call(metric_name='cirros_vnf_memory_util',
 556                                      ns_id='test_nsr_id',
 557                                      operation='LT',
 558                                      statistic='AVERAGE',
 559                                      threshold=20.0,
 560                                      vdu_name='cirros_ns-2-cirros_vnfd-VM-1',
 561                                      vnf_member_index='2')
 562
 563
 564 if __name__ == '__main__':
 565     unittest.main()