osm_policy_module/tests/integration/test_policy_agent.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Whitestack, LLC
   4 # *************************************************************
   5
   6 # This file is part of OSM Monitoring module
   7 # All Rights Reserved to Whitestack, LLC
   8
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20
  21 # For those usages not covered by the Apache License, Version 2.0 please
  22 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  23 ##
  24 import asyncio
  25 import logging
  26 import os
  27 import sys
  28 import unittest
  29 import uuid
  30 from unittest.mock import patch, Mock
  31
  32 from kafka import KafkaProducer
  33 from osm_common.dbmongo import DbMongo
  34 from playhouse.db_url import connect
  35
  36 from osm_policy_module.common.common_db_client import CommonDbClient
  37 from osm_policy_module.common.mon_client import MonClient
  38 from osm_policy_module.core import database
  39 from osm_policy_module.core.agent import PolicyModuleAgent
  40 from osm_policy_module.core.config import Config
  41 from osm_policy_module.core.database import ScalingGroup, ScalingAlarm, ScalingPolicy, ScalingCriteria, VnfAlarm, \
  42     AlarmAction
  43
  44 log = logging.getLogger()
  45 log.level = logging.INFO
  46 stream_handler = logging.StreamHandler(sys.stdout)
  47 log.addHandler(stream_handler)
  48
  49 nsr_record_mock = {
  50     "_id": "87776f33-b67c-417a-8119-cb08e4098951",
  51     "crete-time": 1535392482.0044956,
  52     "operational-status": "running",
  53     "ssh-authorized-key": None,
  54     "name-ref": "cirros_ns",
  55     "nsd": {
  56         "_id": "d7c8bd3c-eb39-4514-8847-19f01345524f",
  57         "_admin": {
  58             "created": 1535392246.499733,
  59             "userDefinedData": {
  60
  61             },
  62             "usageSate": "NOT_IN_USE",
  63             "storage": {
  64                 "zipfile": "package.tar.gz",
  65                 "fs": "local",
  66                 "path": "/app/storage/",
  67                 "folder": "d7c8bd3c-eb39-4514-8847-19f01345524f",
  68                 "pkg-dir": "cirros_nsd",
  69                 "descriptor": "cirros_nsd/cirros_vdu_scaling_nsd.yaml"
  70             },
  71             "onboardingState": "ONBOARDED",
  72             "modified": 1535392246.499733,
  73             "projects_read": [
  74                 "admin"
  75             ],
  76             "operationalState": "ENABLED",
  77             "projects_write": [
  78                 "admin"
  79             ]
  80         },
  81         "id": "cirros_vdu_scaling_ns",
  82         "name": "cirros_vdu_scaling_ns",
  83         "description": "Simple NS example with a cirros_vdu_scaling_vnf",
  84         "designer": "OSM", "version": "1.0",
  85         "vnfd-id": ["cirros_vdu_scaling_vnf"],
  86         "df": [{
  87             "id": "default-df",
  88             "vnf-profile": [
  89                 {
  90                     "id": "1",
  91                     "vnfd-id": "cirros_vdu_scaling_vnf",
  92                     "virtual-link-connectivity": [{
  93                         "virtual-link-profile-id": "cirros_nsd_vld1",
  94                         "constituent-cpd-id": [{
  95                             "constituent-base-element-id": "1",
  96                             "constituent-cpd-id": "eth0-ext"
  97                         }]
  98                     }]
  99                 }, {
 100                     "id": "2",
 101                     "vnfd-id": "cirros_vdu_scaling_vnf",
 102                     "virtual-link-connectivity": [{
 103                         "virtual-link-profile-id": "cirros_nsd_vld1",
 104                         "constituent-cpd-id": [{
 105                             "constituent-base-element-id": "2",
 106                             "constituent-cpd-id": "eth0-ext"
 107                         }]
 108                     }]
 109                 }
 110             ]
 111         }],
 112         "virtual-link-desc": [{
 113             "id": "cirros_nsd_vld1", "mgmt-network": "true"
 114         }]
 115     },
 116     "id": "87776f33-b67c-417a-8119-cb08e4098951",
 117     "config-status": "configured",
 118     "operational-events": [],
 119     "_admin": {
 120         "created": 1535392482.0084584,
 121         "projects_read": [
 122             "admin"
 123         ],
 124         "nsState": "INSTANTIATED",
 125         "modified": 1535392482.0084584,
 126         "projects_write": [
 127             "admin"
 128         ],
 129         "deployed": {
 130             "RO": {
 131                 "vnfd_id": {
 132                     "cirros_vdu_scaling_vnf": "7445e347-fe2f-431a-abc2-8b9be3d093c6"
 133                 },
 134                 "nsd_id": "92c56cf0-f8fa-488c-9afb-9f3d78ae6bbb",
 135                 "nsr_id": "637e12cd-c201-4c44-8ebd-70fb57a4dcee",
 136                 "nsr_status": "BUILD"
 137             }
 138         }
 139     },
 140     "nsd-ref": "cirros_vdu_scaling_ns",
 141     "name": "cirros_ns",
 142     "resource-orchestrator": "osmopenmano",
 143     "instantiate_params": {
 144         "nsDescription": "default description",
 145         "nsdId": "d7c8bd3c-eb39-4514-8847-19f01345524f",
 146         "nsr_id": "87776f33-b67c-417a-8119-cb08e4098951",
 147         "nsName": "cirros_ns",
 148         "vimAccountId": "be48ae31-1d46-4892-a4b4-d69abd55714b"
 149     },
 150     "description": "default description",
 151     "constituent-vnfr-ref": [
 152         "0d9d06ad-3fc2-418c-9934-465e815fafe2",
 153         "3336eb44-77df-4c4f-9881-d2828d259864"
 154     ],
 155     "admin-status": "ENABLED",
 156     "detailed-status": "done",
 157     "datacenter": "be48ae31-1d46-4892-a4b4-d69abd55714b",
 158     "orchestration-progress": {
 159
 160     },
 161     "short-name": "cirros_ns",
 162     "ns-instance-config-ref": "87776f33-b67c-417a-8119-cb08e4098951",
 163     "nsd-name-ref": "cirros_vdu_scaling_ns",
 164     "admin": {
 165         "deployed": {
 166             "RO": {
 167                 "nsr_status": "ACTIVE"
 168             }
 169         }
 170     }
 171 }
 172
 173 vnfr_record_mocks = [
 174     {
 175         "_id": "0d9d06ad-3fc2-418c-9934-465e815fafe2",
 176         "ip-address": "192.168.160.2",
 177         "created-time": 1535392482.0044956,
 178         "vim-account-id": "be48ae31-1d46-4892-a4b4-d69abd55714b",
 179         "vdur": [
 180             {
 181                 "interfaces": [
 182                     {
 183                         "mac-address": "fa:16:3e:71:fd:b8",
 184                         "name": "eth0",
 185                         "ip-address": "192.168.160.2"
 186                     }
 187                 ],
 188                 "status": "ACTIVE",
 189                 "vim-id": "63a65636-9fc8-4022-b070-980823e6266a",
 190                 "name": "cirros_ns-1-cirros_vnfd-VM-1",
 191                 "status-detailed": None,
 192                 "ip-address": "192.168.160.2",
 193                 "vdu-id-ref": "cirros_vnfd-VM"
 194             }
 195         ],
 196         "id": "0d9d06ad-3fc2-418c-9934-465e815fafe2",
 197         "vnfd-ref": "cirros_vdu_scaling_vnf",
 198         "vnfd-id": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 199         "_admin": {
 200             "created": 1535392482.0067868,
 201             "projects_read": [
 202                 "admin"
 203             ],
 204             "modified": 1535392482.0067868,
 205             "projects_write": [
 206                 "admin"
 207             ]
 208         },
 209         "nsr-id-ref": "87776f33-b67c-417a-8119-cb08e4098951",
 210         "member-vnf-index-ref": "1",
 211         "connection-point": [
 212             {
 213                 "name": "eth0",
 214                 "id": None,
 215                 "connection-point-id": None
 216             }
 217         ]
 218     },
 219     {
 220         "_id": "3336eb44-77df-4c4f-9881-d2828d259864",
 221         "ip-address": "192.168.160.10",
 222         "created-time": 1535392482.0044956,
 223         "vim-account-id": "be48ae31-1d46-4892-a4b4-d69abd55714b",
 224         "vdur": [
 225             {
 226                 "interfaces": [
 227                     {
 228                         "mac-address": "fa:16:3e:1e:76:e8",
 229                         "name": "eth0",
 230                         "ip-address": "192.168.160.10"
 231                     }
 232                 ],
 233                 "status": "ACTIVE",
 234                 "vim-id": "a154b8d3-2b10-421a-a51d-4b391d9bd366",
 235                 "name": "cirros_ns-2-cirros_vnfd-VM-1",
 236                 "status-detailed": None,
 237                 "ip-address": "192.168.160.10",
 238                 "vdu-id-ref": "cirros_vnfd-VM"
 239             }
 240         ],
 241         "id": "3336eb44-77df-4c4f-9881-d2828d259864",
 242         "vnfd-ref": "cirros_vdu_scaling_vnf",
 243         "vnfd-id": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 244         "_admin": {
 245             "created": 1535392482.0076294,
 246             "projects_read": [
 247                 "admin"
 248             ],
 249             "modified": 1535392482.0076294,
 250             "projects_write": [
 251                 "admin"
 252             ]
 253         },
 254         "nsr-id-ref": "87776f33-b67c-417a-8119-cb08e4098951",
 255         "member-vnf-index-ref": "2",
 256         "connection-point": [
 257             {
 258                 "name": "eth0",
 259                 "id": None,
 260                 "connection-point-id": None
 261             }
 262         ]}]
 263
 264 nsd_record_mock = {
 265     'id': 'cirros_vdu_scaling_ns',
 266     'name': 'cirros_vdu_scaling_ns',
 267     'description': 'Simple NS example with a cirros_vdu_scaling_vnf',
 268     'designer': 'OSM',
 269     'version': '1.0',
 270     'vnfd-id': ['cirros_vdu_scaling_vnf'],
 271     'df': [{
 272         'id': 'default-df',
 273         'vnf-profile': [
 274             {
 275                 'id': '1',
 276                 'vnfd-id': 'cirros_vdu_scaling_vnf',
 277                 'virtual-link-connectivity': [{
 278                     'virtual-link-profile-id': 'cirros_nsd_vld1',
 279                     'constituent-cpd-id': [{
 280                         'constituent-base-element-id': '1',
 281                         'constituent-cpd-id': 'eth0-ext'
 282                     }]
 283                 }]
 284             }, {
 285                 'id': '2',
 286                 'vnfd-id': 'cirros_vdu_scaling_vnf',
 287                 'virtual-link-connectivity': [{
 288                     'virtual-link-profile-id': 'cirros_nsd_vld1',
 289                     'constituent-cpd-id': [{
 290                         'constituent-base-element-id': '2',
 291                         'constituent-cpd-id': 'eth0-ext'
 292                     }]
 293                 }]
 294             }
 295         ]
 296     }],
 297     'virtual-link-desc': [{
 298         'id': 'cirros_nsd_vld1',
 299         'mgmt-network': 'true'
 300     }]
 301 }
 302
 303
 304 vnfd_record_mock = {
 305     'id': 'cirros_vdu_scaling_vnf',
 306     "_id": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 307     'product-name': 'cirros_vdu_scaling_vnf',
 308     'description': 'Simple VNF example with a cirros and a scaling group descriptor',
 309     'provider': 'OSM',
 310     'version': '1.0',
 311     'mgmt-cp': 'eth0-ext',
 312     'virtual-storage-desc': [{
 313         'id': 'cirros_vnfd-VM-storage',
 314         'size-of-storage': 2
 315     }],
 316     'virtual-compute-desc': [{
 317         'id': 'cirros_vnfd-VM-compute',
 318         'virtual-cpu': {
 319             'num-virtual-cpu': 1
 320         },
 321         'virtual-memory': {
 322             'size': 0.25
 323         }
 324     }],
 325     'sw-image-desc': [{
 326         'id': 'cirros034',
 327         'name': 'cirros034',
 328         'image': 'cirros034'
 329     }],
 330     'vdu': [{
 331         'id': 'cirros_vnfd-VM',
 332         'description': 'cirros_vnfd-VM',
 333         'name': 'cirros_vnfd-VM',
 334         'alarm': [{
 335             'value': 20.0,
 336             'actions': {
 337                 'insufficient-data': [{
 338                     'url': 'localhost:9090'
 339                 }],
 340                 'ok': [{
 341                     'url': 'localhost:9090'
 342                 }],
 343                 'alarm': [{
 344                     'url': 'localhost:9090'
 345                 }]
 346             },
 347             'alarm-id': 'alarm-1',
 348             'operation': 'LT',
 349             'vnf-monitoring-param-ref': 'cirros_vnf_memory_util'
 350         }],
 351         'sw-image-desc': 'cirros034',
 352         'virtual-compute-desc': 'cirros_vnfd-VM-compute',
 353         'virtual-storage-desc': ['cirros_vnfd-VM-storage'],
 354         'int-cpd': [{
 355             'id': 'eth0-int',
 356             'virtual-network-interface-requirement': [{
 357                 'name': 'eth0',
 358                 'virtual-interface': {
 359                     'bandwidth': '0',
 360                     'type': 'VIRTIO',
 361                     'vpci': '0000:00:0a.0'}
 362             }]
 363         }],
 364         'monitoring-parameter': [{
 365             'id': 'cirros_vnf_memory_util',
 366             'name': 'cirros_vnf_memory_util',
 367             'performance-metric': 'average_memory_utilization'
 368         }]
 369     }],
 370     'df': [{
 371         'id': 'default-df',
 372         'vdu-profile': [{
 373             'id': 'cirros_vnfd-VM',
 374             'min-number-of-instances': 1,
 375             'max-number-of-instances': 10,
 376             'vdu-configuration-id': 'cirros_vnfd-VM-vdu-configuration'
 377         }],
 378         'instantiation-level': [{
 379             'id': 'default-instantiation-level',
 380             'vdu-level': [{
 381                 'vdu-id': 'cirros_vnfd-VM',
 382                 'number-of-instances': 1}]
 383         }],
 384         'scaling-aspect': [{
 385             'id': 'scale_cirros_vnfd-VM',
 386             'name': 'scale_cirros_vnfd-VM',
 387             'max-scale-level': 10,
 388             'scaling-policy': [{
 389                 'name': 'auto_memory_util_above_threshold',
 390                 'scaling-type': 'automatic',
 391                 'cooldown-time': 60,
 392                 'threshold-time': 10,
 393                 'scaling-criteria': [{
 394                     'name': 'group1_memory_util_above_threshold',
 395                     'vnf-monitoring-param-ref': 'cirros_vnf_memory_util',
 396                     'scale-out-threshold': 80,
 397                     'scale-out-relational-operation': 'GT',
 398                     'scale-in-relational-operation': 'LT',
 399                     'scale-in-threshold': 20
 400                 }]
 401             }],
 402             'aspect-delta-details': {
 403                 'deltas': [{
 404                     'id': 'scale_cirros_vnfd-VM-delta',
 405                     'vdu-delta': [{
 406                         'number-of-instances': 1,
 407                         'id': 'cirros_vnfd-VM'}]
 408                 }]
 409             }
 410         }]
 411     }],
 412     'ext-cpd': [{
 413         'id': 'eth0-ext',
 414         'int-cpd': {
 415             'vdu-id': 'cirros_vnfd-VM',
 416             'cpd': 'eth0-int'
 417         }
 418     }],
 419     'vdu-configuration': [{
 420         'juju': {
 421             'charm': 'testmetrics',
 422             'proxy': True
 423         },
 424         'metrics': [{
 425             'name': 'users'
 426         }],
 427         'id': 'cirros_vnfd-VM-vdu-configuration'
 428     }],
 429     "_admin": {
 430         "created": 1535392242.6281035,
 431         "modified": 1535392242.6281035,
 432         "storage": {
 433             "zipfile": "package.tar.gz",
 434             "pkg-dir": "cirros_vnf",
 435             "path": "/app/storage/",
 436             "folder": "63f44c41-45ee-456b-b10d-5f08fb1796e0",
 437             "fs": "local",
 438             "descriptor": "cirros_vnf/cirros_vdu_scaling_vnfd.yaml"
 439         },
 440         "usageSate": "NOT_IN_USE",
 441         "onboardingState": "ONBOARDED",
 442         "userDefinedData": {
 443
 444         },
 445         "projects_read": [
 446             "admin"
 447         ],
 448         "operationalState": "ENABLED",
 449         "projects_write": [
 450             "admin"
 451         ]
 452     }
 453 }
 454
 455 MODELS = [ScalingGroup, ScalingPolicy, ScalingCriteria, ScalingAlarm, VnfAlarm, AlarmAction]
 456
 457
 458 class PolicyModuleAgentTest(unittest.TestCase):
 459     def setUp(self):
 460         super()
 461         database.db.initialize(connect('sqlite:///test_db.sqlite'))
 462         database.db.bind(MODELS)
 463         database.db.connect()
 464         database.db.drop_tables(MODELS)
 465         database.db.create_tables(MODELS)
 466         database.db.close()
 467         self.loop = asyncio.new_event_loop()
 468
 469     def tearDown(self):
 470         super()
 471         os.remove('test_db.sqlite')
 472
 473     @patch.object(DbMongo, 'db_connect', Mock())
 474     @patch.object(KafkaProducer, '__init__')
 475     @patch.object(MonClient, 'create_alarm')
 476     @patch.object(CommonDbClient, 'get_vnfd')
 477     @patch.object(CommonDbClient, 'get_nsr')
 478     @patch.object(CommonDbClient, 'get_vnfr')
 479     def test_configure_scaling_groups(self, get_vnfr, get_nsr, get_vnfd, create_alarm, kafka_producer_init):
 480         def _test_configure_scaling_groups_get_vnfr(*args, **kwargs):
 481             if '1' in args[1]:
 482                 return vnfr_record_mocks[0]
 483             if '2' in args[1]:
 484                 return vnfr_record_mocks[1]
 485
 486         def assert_not_called_with(*args, **kwargs):
 487             try:
 488                 create_alarm.assert_called_with(*args, **kwargs)
 489             except AssertionError:
 490                 return
 491             raise AssertionError('Expected to not have been called.')
 492
 493         async def _test_configure_scaling_groups_create_alarm(*args, **kwargs):
 494             return uuid.uuid4()
 495
 496         kafka_producer_init.return_value = None
 497         get_vnfr.side_effect = _test_configure_scaling_groups_get_vnfr
 498         get_nsr.return_value = nsr_record_mock
 499         get_vnfd.return_value = vnfd_record_mock
 500         create_alarm.side_effect = _test_configure_scaling_groups_create_alarm
 501         create_alarm.assert_not_called_with = assert_not_called_with
 502         config = Config()
 503         agent = PolicyModuleAgent(config, self.loop)
 504         self.loop.run_until_complete(agent.autoscaling_service.configure_scaling_groups("test_nsr_id"))
 505         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 506                                      ns_id='test_nsr_id',
 507                                      operation='GT',
 508                                      threshold=80,
 509                                      vdu_name='cirros_ns-1-cirros_vnfd-VM-1',
 510                                      vnf_member_index='1')
 511         create_alarm.assert_not_called_with(metric_name='average_memory_utilization',
 512                                             ns_id='test_nsr_id',
 513                                             operation='LT',
 514                                             threshold=20,
 515                                             vdu_name='cirros_ns-1-cirros_vnfd-VM-1',
 516                                             vnf_member_index='1')
 517         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 518                                      ns_id='test_nsr_id',
 519                                      operation='GT',
 520                                      threshold=80,
 521                                      vdu_name='cirros_ns-2-cirros_vnfd-VM-1',
 522                                      vnf_member_index='2')
 523         create_alarm.assert_not_called_with(metric_name='average_memory_utilization',
 524                                             ns_id='test_nsr_id',
 525                                             operation='LT',
 526                                             threshold=20,
 527                                             vdu_name='cirros_ns-2-cirros_vnfd-VM-1',
 528                                             vnf_member_index='2')
 529         scaling_record = ScalingGroup.get()
 530         self.assertEqual(scaling_record.name, 'scale_cirros_vnfd-VM')
 531         self.assertEqual(scaling_record.nsr_id, 'test_nsr_id')
 532
 533     @patch.object(DbMongo, 'db_connect', Mock())
 534     @patch.object(KafkaProducer, '__init__')
 535     @patch.object(MonClient, 'create_alarm')
 536     @patch.object(CommonDbClient, 'get_vnfd')
 537     @patch.object(CommonDbClient, 'get_nsr')
 538     @patch.object(CommonDbClient, 'get_vnfr')
 539     def test_configure_vnf_alarms(self, get_vnfr, get_nsr, get_vnfd, create_alarm, kafka_producer_init):
 540         def _test_configure_scaling_groups_get_vnfr(*args, **kwargs):
 541             if '1' in args[1]:
 542                 return vnfr_record_mocks[0]
 543             if '2' in args[1]:
 544                 return vnfr_record_mocks[1]
 545
 546         async def _test_configure_vnf_alarms_create_alarm(*args, **kwargs):
 547             return uuid.uuid4()
 548
 549         kafka_producer_init.return_value = None
 550         get_vnfr.side_effect = _test_configure_scaling_groups_get_vnfr
 551         get_nsr.return_value = nsr_record_mock
 552         get_vnfd.return_value = vnfd_record_mock
 553         create_alarm.side_effect = _test_configure_vnf_alarms_create_alarm
 554         config = Config()
 555         agent = PolicyModuleAgent(config, self.loop)
 556         self.loop.run_until_complete(agent.alarming_service.configure_vnf_alarms("test_nsr_id"))
 557         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 558                                      ns_id='test_nsr_id',
 559                                      vdu_name='cirros_ns-1-cirros_vnfd-VM-1',
 560                                      vnf_member_index='1',
 561                                      threshold=20.0,
 562                                      operation='LT')
 563         create_alarm.assert_any_call(metric_name='average_memory_utilization',
 564                                      ns_id='test_nsr_id',
 565                                      vdu_name='cirros_ns-2-cirros_vnfd-VM-1',
 566                                      vnf_member_index='2',
 567                                      threshold=20.0,
 568                                      operation='LT')
 569
 570
 571 if __name__ == '__main__':
 572     unittest.main()