osm_policy_module/tests/integration/test_policy_agent.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Whitestack, LLC
   4 # *************************************************************
   5
   6 # This file is part of OSM Monitoring module
   7 # All Rights Reserved to Whitestack, LLC
   8
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  23 ##
  24 import asyncio
  25 import logging
  26 import os
  27 import sys
  28 import unittest
  29 import uuid
  30 from unittest.mock import patch, Mock
  31
  32 from kafka import KafkaProducer
  33 from osm_common.dbmongo import DbMongo
  34 from playhouse.db_url import connect
  35
  36 from osm_policy_module.common.common_db_client import CommonDbClient
  37 from osm_policy_module.common.mon_client import MonClient
  38 from osm_policy_module.core import database
  39 from osm_policy_module.core.agent import PolicyModuleAgent
  40 from osm_policy_module.core.config import Config
  41 from osm_policy_module.core.database import ScalingGroup, ScalingAlarm, ScalingPolicy, ScalingCriteria, VnfAlarm, \
  42     AlarmAction
  43
  44 log = logging.getLogger()
  45 log.level = logging.INFO
  46 stream_handler = logging.StreamHandler(sys.stdout)
  47 log.addHandler(stream_handler)
  48
  49 nsr_record_mock = {
  50     "_id": "87776f33-b67c-417a-8119-cb08e4098951",
  51     "crete-time": 1535392482.0044956,
  52     "operational-status": "running",
  53     "ssh-authorized-key": None,
  54     "name-ref": "cirros_ns",
  55     "nsd": {
  56         "name": "cirros_vdu_scaling_ns",
  57         "_id": "d7c8bd3c-eb39-4514-8847-19f01345524f",
  58         "vld": [
  59             {
  60                 "id": "cirros_nsd_vld1",
  61                 "name": "cirros_nsd_vld1",
  62                 "type": "ELAN",
  63                 "mgmt-network": "true",
  64                 "vnfd-connection-point-ref": [
  65                     {
  66                         "vnfd-id-ref": "cirros_vdu_scaling_vnf",
  67                         "member-vnf-index-ref": 1,
  68                         "vnfd-connection-point-ref": "eth0"
  69                     },
  70                     {
  71                         "vnfd-id-ref": "cirros_vdu_scaling_vnf",
  72                         "member-vnf-index-ref": 2,
  73                         "vnfd-connection-point-ref": "eth0"
  74                     }
  75                 ]
  76             }
  77         ],
  78         "vendor": "OSM",
  79         "constituent-vnfd": [
  80             {
  81                 "member-vnf-index": "1",
  82                 "vnfd-id-ref": "cirros_vdu_scaling_vnf"
  83             },
  84             {
  85                 "member-vnf-index": "2",
  86                 "vnfd-id-ref": "cirros_vdu_scaling_vnf"
  87             }
  88         ],
  89         "version": "1.0",
  90         "id": "cirros_vdu_scaling_ns",
  91         "description": "Simple NS example with a cirros_vdu_scaling_vnf",
  92         "logo": "osm.png",
  93         "_admin": {
  94             "created": 1535392246.499733,
  95             "userDefinedData": {
  96
  97             },
  98             "usageSate": "NOT_IN_USE",
  99             "storage": {
 100                 "zipfile": "package.tar.gz",
 101                 "fs": "local",
 102                 "path": "/app/storage/",
 103                 "folder": "d7c8bd3c-eb39-4514-8847-19f01345524f",
 104                 "pkg-dir": "cirros_nsd",
 105                 "descriptor": "cirros_nsd/cirros_vdu_scaling_nsd.yaml"
 106             },
 107             "onboardingState": "ONBOARDED",
 108             "modified": 1535392246.499733,
 109             "projects_read": [
 110                 "admin"
 111             ],
 112             "operationalState": "ENABLED",
 113             "projects_write": [
 114                 "admin"
 115             ]
 116         },
 117         "short-name": "cirros_vdu_scaling_ns"
 118     },
 119     "id": "87776f33-b67c-417a-8119-cb08e4098951",
 120     "config-status": "configured",
 121     "operational-events": [],
 122     "_admin": {
 123         "created": 1535392482.0084584,
 124         "projects_read": [
 125             "admin"
 126         ],
 127         "nsState": "INSTANTIATED",
 128         "modified": 1535392482.0084584,
 129         "projects_write": [
 130             "admin"
 131         ],
 132         "deployed": {
 133             "RO": {
 134                 "vnfd_id": {
 135                     "cirros_vdu_scaling_vnf": "7445e347-fe2f-431a-abc2-8b9be3d093c6"
 136                 },
 137                 "nsd_id": "92c56cf0-f8fa-488c-9afb-9f3d78ae6bbb",
 138                 "nsr_id": "637e12cd-c201-4c44-8ebd-70fb57a4dcee",
 139                 "nsr_status": "BUILD"
 140             }
 141         }
 142     },
 143     "nsd-ref": "cirros_vdu_scaling_ns",
 144     "name": "cirros_ns",
 145     "resource-orchestrator": "osmopenmano",
 146     "instantiate_params": {
 147         "nsDescription": "default description",
 148         "nsdId": "d7c8bd3c-eb39-4514-8847-19f01345524f",
 149         "nsr_id": "87776f33-b67c-417a-8119-cb08e4098951",
 150         "nsName": "cirros_ns",
 151         "vimAccountId": "be48ae31-1d46-4892-a4b4-d69abd55714b"
 152     },
 153     "description": "default description",
 154     "constituent-vnfr-ref": [
 155         "0d9d06ad-3fc2-418c-9934-465e815fafe2",
 156         "3336eb44-77df-4c4f-9881-d2828d259864"
 157     ],
 158     "admin-status": "ENABLED",
 159     "detailed-status": "done",
 160     "datacenter": "be48ae31-1d46-4892-a4b4-d69abd55714b",
 161     "orchestration-progress": {
 162
 163     },
 164     "short-name": "cirros_ns",
 165     "ns-instance-config-ref": "87776f33-b67c-417a-8119-cb08e4098951",
 166     "nsd-name-ref": "cirros_vdu_scaling_ns",
 167     "admin": {
 168         "deployed": {
 169             "RO": {
 170                 "nsr_status": "ACTIVE"
 171             }
 172         }
 173     }
 174 }
 175
 176 vnfr_record_mocks = [
 177     {
 178         "_id": "0d9d06ad-3fc2-418c-9934-465e815fafe2",
 179         "ip-address": "192.168.160.2",
 180         "created-time": 1535392482.0044956,
 181         "vim-account-id": "be48ae31-1d46-4892-a4b4-d69abd55714b",
 182         "vdur": [
 183             {
 184                 "interfaces": [
 185                     {
 186                         "mac-address": "fa:16:3e:71:fd:b8",
 187                         "name": "eth0",
 188                         "ip-address": "192.168.160.2"
 189                     }
 190                 ],
 191                 "status": "ACTIVE",
 192                 "vim-id": "63a65636-9fc8-4022-b070-980823e6266a",
 193                 "name": "cirros_ns-1-cirros_vnfd-VM-1",
 194                 "status-detailed": None,
 195                 "ip-address": "192.168.160.2",
 196                 "vdu-id-ref": "cirros_vnfd-VM"
 197             }
 198         ],
 199         "id": "0d9d06ad-3fc2-418c-9934-465e815fafe2",
 200         "vnfd-ref": "cirros_vdu_scaling_vnf",
 201         "vnfd-id": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 202         "_admin": {
 203             "created": 1535392482.0067868,
 204             "projects_read": [
 205                 "admin"
 206             ],
 207             "modified": 1535392482.0067868,
 208             "projects_write": [
 209                 "admin"
 210             ]
 211         },
 212         "nsr-id-ref": "87776f33-b67c-417a-8119-cb08e4098951",
 213         "member-vnf-index-ref": "1",
 214         "connection-point": [
 215             {
 216                 "name": "eth0",
 217                 "id": None,
 218                 "connection-point-id": None
 219             }
 220         ]
 221     },
 222     {
 223         "_id": "3336eb44-77df-4c4f-9881-d2828d259864",
 224         "ip-address": "192.168.160.10",
 225         "created-time": 1535392482.0044956,
 226         "vim-account-id": "be48ae31-1d46-4892-a4b4-d69abd55714b",
 227         "vdur": [
 228             {
 229                 "interfaces": [
 230                     {
 231                         "mac-address": "fa:16:3e:1e:76:e8",
 232                         "name": "eth0",
 233                         "ip-address": "192.168.160.10"
 234                     }
 235                 ],
 236                 "status": "ACTIVE",
 237                 "vim-id": "a154b8d3-2b10-421a-a51d-4b391d9bd366",
 238                 "name": "cirros_ns-2-cirros_vnfd-VM-1",
 239                 "status-detailed": None,
 240                 "ip-address": "192.168.160.10",
 241                 "vdu-id-ref": "cirros_vnfd-VM"
 242             }
 243         ],
 244         "id": "3336eb44-77df-4c4f-9881-d2828d259864",
 245         "vnfd-ref": "cirros_vdu_scaling_vnf",
 246         "vnfd-id": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 247         "_admin": {
 248             "created": 1535392482.0076294,
 249             "projects_read": [
 250                 "admin"
 251             ],
 252             "modified": 1535392482.0076294,
 253             "projects_write": [
 254                 "admin"
 255             ]
 256         },
 257         "nsr-id-ref": "87776f33-b67c-417a-8119-cb08e4098951",
 258         "member-vnf-index-ref": "2",
 259         "connection-point": [
 260             {
 261                 "name": "eth0",
 262                 "id": None,
 263                 "connection-point-id": None
 264             }
 265         ]}]
 266
 267 nsd_record_mock = {'name': 'cirros_vdu_scaling_ns',
 268                    'version': '1.0',
 269                    'short-name': 'cirros_vdu_scaling_ns',
 270                    'logo': 'osm.png',
 271                    'id': 'cirros_vdu_scaling_ns',
 272                    'description': 'Simple NS example with a cirros_vdu_scaling_vnf',
 273                    'vendor': 'OSM',
 274                    'vld': [
 275                        {'name': 'cirros_nsd_vld1',
 276                         'type': 'ELAN',
 277                         'id': 'cirros_nsd_vld1',
 278                         'mgmt-network': 'true',
 279                         'vnfd-connection-point-ref': [
 280                             {'vnfd-id-ref': 'cirros_vdu_scaling_vnf',
 281                              'vnfd-connection-point-ref': 'eth0',
 282                              'member-vnf-index-ref': 1},
 283                             {'vnfd-id-ref': 'cirros_vdu_scaling_vnf',
 284                              'vnfd-connection-point-ref': 'eth0',
 285                              'member-vnf-index-ref': 2}]}],
 286                    'constituent-vnfd': [{'vnfd-id-ref': 'cirros_vdu_scaling_vnf',
 287                                          'member-vnf-index': '1'},
 288                                         {'vnfd-id-ref': 'cirros_vdu_scaling_vnf',
 289                                          'member-vnf-index': '2'}]}
 290
 291 vnfd_record_mock = {
 292     "_id": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 293     "name": "cirros_vdu_scaling_vnf",
 294     "vendor": "OSM",
 295     "vdu": [
 296         {
 297             "name": "cirros_vnfd-VM",
 298             "monitoring-param": [
 299                 {
 300                     "id": "cirros_vnfd-VM_memory_util",
 301                     "nfvi-metric": "average_memory_utilization"
 302                 }
 303             ],
 304             "vm-flavor": {
 305                 "vcpu-count": 1,
 306                 "memory-mb": 256,
 307                 "storage-gb": 2
 308             },
 309             "description": "cirros_vnfd-VM",
 310             "count": 1,
 311             "id": "cirros_vnfd-VM",
 312             "interface": [
 313                 {
 314                     "name": "eth0",
 315                     "external-connection-point-ref": "eth0",
 316                     "type": "EXTERNAL",
 317                     "virtual-interface": {
 318                         "bandwidth": "0",
 319                         "type": "VIRTIO",
 320                         "vpci": "0000:00:0a.0"
 321                     }
 322                 }
 323             ],
 324             "image": "cirros034",
 325             "alarm": [
 326                 {
 327                     "value": 20.0000,
 328                     "actions": {
 329                         "insufficient-data": [
 330                             {
 331                                 "url": "localhost:9090"
 332                             }
 333                         ],
 334                         "ok": [
 335                             {
 336                                 "url": "localhost:9090"
 337                             }
 338                         ],
 339                         "alarm": [
 340                             {
 341                                 "url": "localhost:9090"
 342                             }
 343                         ]
 344                     },
 345                     "alarm-id": "alarm-1",
 346                     "operation": "LT",
 347                     "vnf-monitoring-param-ref": "cirros_vnf_memory_util"
 348                 }
 349             ]
 350         }
 351     ],
 352     "monitoring-param": [
 353         {
 354             "id": "cirros_vnf_memory_util",
 355             "name": "cirros_vnf_memory_util",
 356             "aggregation-type": "AVERAGE",
 357             "vdu-monitoring-param": {
 358                 "vdu-monitoring-param-ref": "cirros_vnfd-VM_memory_util",
 359                 "vdu-ref": "cirros_vnfd-VM"
 360             }
 361         },
 362         {
 363             "id": "haproxy_users",
 364             "name": "haproxy_users",
 365             "aggregation-type": "AVERAGE",
 366             "vnf-metric":
 367                 {"vnf-metric-name-ref": "users"}
 368         }
 369     ],
 370     "vdu-configuration": {
 371         "juju": {
 372             "charm": "testmetrics",
 373             "proxy": True
 374         },
 375         "metrics": {
 376             "name": "users"
 377         }
 378     },
 379     "description": "Simple VNF example with a cirros and a scaling group descriptor",
 380     "id": "cirros_vdu_scaling_vnf",
 381     "logo": "cirros-64.png",
 382     "version": "1.0",
 383     "connection-point": [
 384         {
 385             "name": "eth0",
 386             "type": "VPORT"
 387         }
 388     ],
 389     "mgmt-interface": {
 390         "cp": "eth0"
 391     },
 392     "scaling-group-descriptor": [
 393         {
 394             "name": "scale_cirros_vnfd-VM",
 395             "min-instance-count": 1,
 396             "vdu": [
 397                 {
 398                     "count": 1,
 399                     "vdu-id-ref": "cirros_vnfd-VM"
 400                 }
 401             ],
 402             "max-instance-count": 10,
 403             "scaling-policy": [
 404                 {
 405                     "name": "auto_memory_util_above_threshold",
 406                     "scaling-type": "automatic",
 407                     "cooldown-time": 60,
 408                     "threshold-time": 10,
 409                     "scaling-criteria": [
 410                         {
 411                             "name": "group1_memory_util_above_threshold",
 412                             "vnf-monitoring-param-ref": "cirros_vnf_memory_util",
 413                             "scale-out-threshold": 80,
 414                             "scale-out-relational-operation": "GT",
 415                             "scale-in-relational-operation": "LT",
 416                             "scale-in-threshold": 20
 417                         }
 418                     ]
 419                 }
 420             ]
 421         }
 422     ],
 423     "short-name": "cirros_vdu_scaling_vnf",
 424     "_admin": {
 425         "created": 1535392242.6281035,
 426         "modified": 1535392242.6281035,
 427         "storage": {
 428             "zipfile": "package.tar.gz",
 429             "pkg-dir": "cirros_vnf",
 430             "path": "/app/storage/",
 431             "folder": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 432             "fs": "local",
 433             "descriptor": "cirros_vnf/cirros_vdu_scaling_vnfd.yaml"
 434         },
 435         "usageSate": "NOT_IN_USE",
 436         "onboardingState": "ONBOARDED",
 437         "userDefinedData": {
 438
 439         },
 440         "projects_read": [
 441             "admin"
 442         ],
 443         "operationalState": "ENABLED",
 444         "projects_write": [
 445             "admin"
 446         ]
 447     }
 448 }
 449
 450 MODELS = [ScalingGroup, ScalingPolicy, ScalingCriteria, ScalingAlarm, VnfAlarm, AlarmAction]
 451
 452
 453 class PolicyModuleAgentTest(unittest.TestCase):
 454     def setUp(self):
 455         super()
 456         database.db.initialize(connect('sqlite:///test_db.sqlite'))
 457         database.db.bind(MODELS)
 458         database.db.connect()
 459         database.db.drop_tables(MODELS)
 460         database.db.create_tables(MODELS)
 461         database.db.close()
 462         self.loop = asyncio.new_event_loop()
 463
 464     def tearDown(self):
 465         super()
 466         os.remove('test_db.sqlite')
 467
 468     @patch.object(DbMongo, 'db_connect', Mock())
 469     @patch.object(KafkaProducer, '__init__')
 470     @patch.object(MonClient, 'create_alarm')
 471     @patch.object(CommonDbClient, 'get_vnfd')
 472     @patch.object(CommonDbClient, 'get_nsr')
 473     @patch.object(CommonDbClient, 'get_vnfr')
 474     def test_configure_scaling_groups(self, get_vnfr, get_nsr, get_vnfd, create_alarm, kafka_producer_init):
 475         def _test_configure_scaling_groups_get_vnfr(*args, **kwargs):
 476             if '1' in args[1]:
 477                 return vnfr_record_mocks[0]
 478             if '2' in args[1]:
 479                 return vnfr_record_mocks[1]
 480
 481         def assert_not_called_with(*args, **kwargs):
 482             try:
 483                 create_alarm.assert_called_with(*args, **kwargs)
 484             except AssertionError:
 485                 return
 486             raise AssertionError('Expected to not have been called.')
 487
 488         async def _test_configure_scaling_groups_create_alarm(*args, **kwargs):
 489             return uuid.uuid4()
 490
 491         kafka_producer_init.return_value = None
 492         get_vnfr.side_effect = _test_configure_scaling_groups_get_vnfr
 493         get_nsr.return_value = nsr_record_mock
 494         get_vnfd.return_value = vnfd_record_mock
 495         create_alarm.side_effect = _test_configure_scaling_groups_create_alarm
 496         create_alarm.assert_not_called_with = assert_not_called_with
 497         config = Config()
 498         agent = PolicyModuleAgent(config, self.loop)
 499         self.loop.run_until_complete(agent.autoscaling_service.configure_scaling_groups("test_nsr_id"))
 500         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 501                                      ns_id='test_nsr_id',
 502                                      operation='GT',
 503                                      statistic='AVERAGE',
 504                                      threshold=80,
 505                                      vdu_name='cirros_ns-1-cirros_vnfd-VM-1',
 506                                      vnf_member_index='1')
 507         create_alarm.assert_not_called_with(metric_name='average_memory_utilization',
 508                                             ns_id='test_nsr_id',
 509                                             operation='LT',
 510                                             statistic='AVERAGE',
 511                                             threshold=20,
 512                                             vdu_name='cirros_ns-1-cirros_vnfd-VM-1',
 513                                             vnf_member_index='1')
 514         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 515                                      ns_id='test_nsr_id',
 516                                      operation='GT',
 517                                      statistic='AVERAGE',
 518                                      threshold=80,
 519                                      vdu_name='cirros_ns-2-cirros_vnfd-VM-1',
 520                                      vnf_member_index='2')
 521         create_alarm.assert_not_called_with(metric_name='average_memory_utilization',
 522                                             ns_id='test_nsr_id',
 523                                             operation='LT',
 524                                             statistic='AVERAGE',
 525                                             threshold=20,
 526                                             vdu_name='cirros_ns-2-cirros_vnfd-VM-1',
 527                                             vnf_member_index='2')
 528         scaling_record = ScalingGroup.get()
 529         self.assertEqual(scaling_record.name, 'scale_cirros_vnfd-VM')
 530         self.assertEqual(scaling_record.nsr_id, 'test_nsr_id')
 531
 532     @patch.object(DbMongo, 'db_connect', Mock())
 533     @patch.object(KafkaProducer, '__init__')
 534     @patch.object(MonClient, 'create_alarm')
 535     @patch.object(CommonDbClient, 'get_vnfd')
 536     @patch.object(CommonDbClient, 'get_nsr')
 537     @patch.object(CommonDbClient, 'get_vnfr')
 538     def test_configure_vnf_alarms(self, get_vnfr, get_nsr, get_vnfd, create_alarm, kafka_producer_init):
 539         def _test_configure_scaling_groups_get_vnfr(*args, **kwargs):
 540             if '1' in args[1]:
 541                 return vnfr_record_mocks[0]
 542             if '2' in args[1]:
 543                 return vnfr_record_mocks[1]
 544
 545         async def _test_configure_vnf_alarms_create_alarm(*args, **kwargs):
 546             return uuid.uuid4()
 547
 548         kafka_producer_init.return_value = None
 549         get_vnfr.side_effect = _test_configure_scaling_groups_get_vnfr
 550         get_nsr.return_value = nsr_record_mock
 551         get_vnfd.return_value = vnfd_record_mock
 552         create_alarm.side_effect = _test_configure_vnf_alarms_create_alarm
 553         config = Config()
 554         agent = PolicyModuleAgent(config, self.loop)
 555         self.loop.run_until_complete(agent.alarming_service.configure_vnf_alarms("test_nsr_id"))
 556         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 557                                      ns_id='test_nsr_id',
 558                                      vdu_name='cirros_ns-1-cirros_vnfd-VM-1',
 559                                      vnf_member_index='1',
 560                                      threshold=20.0,
 561                                      operation='LT',
 562                                      statistic='AVERAGE')
 563         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 564                                      ns_id='test_nsr_id',
 565                                      vdu_name='cirros_ns-2-cirros_vnfd-VM-1',
 566                                      vnf_member_index='2',
 567                                      threshold=20.0,
 568                                      operation='LT',
 569                                      statistic='AVERAGE')
 570
 571
 572 if __name__ == '__main__':
 573     unittest.main()