rwlaunchpad/ra/pytest/ns/ha/test_ha_multiple_failovers.py

   1 #!/usr/bin/env python3
   2 """
   3 #
   4 #   Copyright 2017 RIFT.IO Inc
   5 #
   6 #   Licensed under the Apache License, Version 2.0 (the "License");
   7 #   you may not use this file except in compliance with the License.
   8 #   You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #   Unless required by applicable law or agreed to in writing, software
  13 #   distributed under the License is distributed on an "AS IS" BASIS,
  14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #   See the License for the specific language governing permissions and
  16 #   limitations under the License.
  17 #
  18 """
  19
  20 import gi
  21 import pytest
  22 import random
  23 import time
  24
  25 import rift.auto.mano as mano
  26 import rift.auto.descriptor
  27 from gi.repository.RwKeyspec import quoted_key
  28
  29 from gi.repository import (
  30     RwProjectNsdYang,
  31     RwNsrYang,
  32     RwVnfrYang,
  33     RwVlrYang,
  34     RwCloudYang,
  35     RwConmanYang,
  36 )
  37
  38 @pytest.fixture(scope='module')
  39 def test_project():
  40     return 'project_ha'
  41
  42 @pytest.mark.setup('active-configuration')
  43 @pytest.mark.incremental
  44 class TestMutipleFailoverActiveSetup(object):
  45     def test_create_project_users(self, rbac_user_passwd, user_domain, rw_active_user_proxy, logger,
  46             rw_active_project_proxy, rw_active_rbac_int_proxy, rw_active_conman_proxy, test_project, user_roles):
  47         # Create test users
  48         user_name_pfx = 'user_ha_'
  49         users = []
  50         for idx in range(1, 9):
  51             users.append(user_name_pfx+str(idx))
  52             mano.create_user(rw_active_user_proxy, user_name_pfx+str(idx), rbac_user_passwd, user_domain)
  53
  54         # Create a test project and assign roles to users in the newly created project
  55         logger.debug('Creating project {}'.format(test_project))
  56         mano.create_project(rw_active_conman_proxy, test_project)
  57
  58         for _ in range(8):
  59             role = random.choice(user_roles)
  60             user = users.pop()
  61             logger.debug('Assinging role {} to user {} in project {}'.format(role, user, test_project))
  62             mano.assign_project_role_to_user(rw_active_project_proxy, role, user, test_project, user_domain,
  63                                             rw_active_rbac_int_proxy)
  64
  65     def test_create_cloud_account(self, cloud_account, fmt_prefixed_cloud_xpath, fmt_cloud_xpath, rw_active_cloud_pxy,
  66                                 test_project, logger):
  67         logger.debug('Creating cloud account {} for project {}'.format(cloud_account.name, test_project))
  68         xpath = fmt_prefixed_cloud_xpath.format(project=quoted_key(test_project),
  69                                                 account_name=quoted_key(cloud_account.name))
  70         rw_active_cloud_pxy.replace_config(xpath, cloud_account)
  71         xpath_no_pfx = fmt_cloud_xpath.format(project=quoted_key(test_project),
  72                                               account_name=quoted_key(cloud_account.name))
  73         response =  rw_active_cloud_pxy.get(xpath_no_pfx)
  74         assert response.name == cloud_account.name
  75         assert response.account_type == cloud_account.account_type
  76
  77         rw_active_cloud_pxy.wait_for(fmt_cloud_xpath.format(project=quoted_key(test_project), account_name=quoted_key(
  78         cloud_account.name)) + '/connection-status/status', 'success', timeout=30, fail_on=['failure'])
  79
  80     def test_onboard_descriptors(self, descriptors, test_project, active_mgmt_session, fmt_nsd_catalog_xpath, logger):
  81         # Uploads the descriptors
  82         pingpong_descriptors = descriptors['pingpong']
  83         for descriptor in pingpong_descriptors:
  84             logger.debug('Onboarding descriptor {} for project {}'.format(descriptor, test_project))
  85             rift.auto.descriptor.onboard(active_mgmt_session, descriptor, project=test_project)
  86
  87         # Verify whether the descriptors uploaded successfully
  88         nsd_pxy = active_mgmt_session.proxy(RwProjectNsdYang)
  89         nsd_xpath = fmt_nsd_catalog_xpath.format(project=quoted_key(test_project))
  90         nsd_catalog = nsd_pxy.get_config(nsd_xpath)
  91         assert nsd_catalog
  92
  93     def test_instantiate_nsr(self, fmt_nsd_catalog_xpath, cloud_account, active_mgmt_session, logger, test_project):
  94         nsd_pxy = active_mgmt_session.proxy(RwProjectNsdYang)
  95         rwnsr_pxy = active_mgmt_session.proxy(RwNsrYang)
  96
  97         nsd_xpath = fmt_nsd_catalog_xpath.format(project=quoted_key(test_project))
  98         nsd_catalog = nsd_pxy.get_config(nsd_xpath)
  99         assert nsd_catalog
 100         nsd = nsd_catalog.nsd[0]
 101         nsr = rift.auto.descriptor.create_nsr(cloud_account.name, nsd.name, nsd)
 102
 103         logger.debug('Instantiating NS for project {}'.format(test_project))
 104         rift.auto.descriptor.instantiate_nsr(nsr, rwnsr_pxy, logger, project=test_project)
 105
 106
 107 @pytest.mark.depends('active-configuration')
 108 @pytest.mark.setup('multiple-failovers')
 109 @pytest.mark.incremental
 110 class TestHaMultipleFailovers(object):
 111     def test_ha_multiple_failovers(self, revertive_pref_host, active_confd_host, standby_confd_host, standby_lp_node_obj, active_lp_node_obj, logger,
 112                                         fmt_cloud_xpath, cloud_account, test_project, active_site_name, standby_site_name, standby_mgmt_session, active_mgmt_session, descriptors):
 113         count, failover_count = 1, 10
 114         current_actv_mgmt_session, current_stdby_mgmt_session = active_mgmt_session, standby_mgmt_session
 115         current_actv_lp_node_obj = active_lp_node_obj
 116
 117         descriptor_list = descriptors['haproxy'][::-1] + descriptors['vdud_cfgfile'][::-1]
 118
 119         original_active_as_standby_kwargs = {'revertive_pref_host': revertive_pref_host, 'new_active_ip': standby_confd_host, 'new_active_site': standby_site_name,
 120             'new_standby_ip': active_confd_host, 'new_standby_site': active_site_name}
 121         original_active_as_active_kwargs = {'revertive_pref_host': revertive_pref_host, 'new_active_ip':active_confd_host, 'new_active_site': active_site_name,
 122             'new_standby_ip': standby_confd_host, 'new_standby_site': standby_site_name}
 123
 124         while count <= failover_count:
 125             kwargs = original_active_as_active_kwargs
 126             if count%2 == 1:
 127                 kwargs = original_active_as_standby_kwargs
 128
 129             # upload descriptor
 130             if count not in [5,6,7,8]:
 131                 descriptor = descriptor_list.pop()
 132                 rift.auto.descriptor.onboard(current_actv_mgmt_session, descriptor, project=test_project)
 133
 134             # Collect config, op-data from current active before doing a failover
 135             current_actv_lp_node_obj.session = None
 136             current_actv_lp_node_obj.collect_data()
 137
 138             time.sleep(5)
 139             logger.debug('Failover Iteration - {}. Current standby {} will be the new active'.format(count, current_stdby_mgmt_session.host))
 140             mano.indirect_failover(**kwargs)
 141
 142             last_actv_lp_node_obj = current_actv_lp_node_obj
 143             current_actv_mgmt_session, current_stdby_mgmt_session = active_mgmt_session, standby_mgmt_session
 144             current_actv_lp_node_obj = active_lp_node_obj
 145             if count%2 == 1:
 146                 current_actv_lp_node_obj = standby_lp_node_obj
 147                 current_actv_mgmt_session, current_stdby_mgmt_session = standby_mgmt_session, active_mgmt_session
 148
 149             logger.debug('Waiting for the new active {} to come up'.format(current_actv_mgmt_session.host))
 150             mano.wait_for_standby_to_become_active(current_actv_mgmt_session)
 151
 152             # Wait for NSR to become active
 153             rw_new_active_cloud_pxy = current_actv_mgmt_session.proxy(RwCloudYang)
 154             rwnsr_proxy = current_actv_mgmt_session.proxy(RwNsrYang)
 155
 156             rw_new_active_cloud_pxy.wait_for(
 157                 fmt_cloud_xpath.format(project=quoted_key(test_project), account_name=quoted_key(
 158                     cloud_account.name)) + '/connection-status/status', 'success', timeout=60, fail_on=['failure'])
 159
 160             nsr_opdata = rwnsr_proxy.get(
 161                     '/rw-project:project[rw-project:name={project}]/ns-instance-opdata'.format(
 162                         project=quoted_key(test_project)))
 163             assert nsr_opdata
 164             nsrs = nsr_opdata.nsr
 165
 166             for nsr in nsrs:
 167                 xpath = "/rw-project:project[rw-project:name={project}]/ns-instance-opdata/nsr[ns-instance-config-ref={config_ref}]/config-status".format(
 168                     project=quoted_key(test_project), config_ref=quoted_key(nsr.ns_instance_config_ref))
 169                 rwnsr_proxy.wait_for(xpath, "configured", fail_on=['failed'], timeout=400)
 170
 171             # Collect config, op-data from new active
 172             current_actv_lp_node_obj.session = None
 173             current_actv_lp_node_obj.collect_data()
 174
 175             # Compare data between last active and current active
 176             current_actv_lp_node_obj.compare(last_actv_lp_node_obj)
 177             count += 1
 178
 179
 180 @pytest.mark.depends('multiple-failovers')
 181 @pytest.mark.incremental
 182 class TestHaOperationPostMultipleFailovers(object):
 183     def test_instantiate_nsr(self, fmt_nsd_catalog_xpath, cloud_account, active_mgmt_session, logger, test_project):
 184         """Check if a new NS instantiation goes through after multiple HA failovers.
 185         It uses metadata cfgfile nsd for the instantiation.
 186         There alreasy exists ping pong NS instantiation"""
 187         nsd_pxy = active_mgmt_session.proxy(RwProjectNsdYang)
 188         rwnsr_pxy = active_mgmt_session.proxy(RwNsrYang)
 189
 190         nsd_xpath = fmt_nsd_catalog_xpath.format(project=quoted_key(test_project))
 191         nsd_catalog = nsd_pxy.get_config(nsd_xpath)
 192         assert nsd_catalog
 193         cfgfile_nsd = [nsd for nsd in nsd_catalog.nsd if 'cfgfile_nsd' in nsd.name][0]
 194         nsr = rift.auto.descriptor.create_nsr(cloud_account.name, cfgfile_nsd.name, cfgfile_nsd)
 195
 196         logger.debug('Instantiating cfgfile NS for project {}'.format(test_project))
 197         rift.auto.descriptor.instantiate_nsr(nsr, rwnsr_pxy, logger, project=test_project)
 198
 199     def test_nsr_terminate(self, active_mgmt_session, logger, test_project):
 200         """"""
 201         rwnsr_pxy = active_mgmt_session.proxy(RwNsrYang)
 202         rwvnfr_pxy = active_mgmt_session.proxy(RwVnfrYang)
 203         rwvlr_pxy = active_mgmt_session.proxy(RwVlrYang)
 204
 205         logger.debug("Trying to terminate ping pong, cfgfile NSRs in project {}".format(test_project))
 206         rift.auto.descriptor.terminate_nsr(rwvnfr_pxy, rwnsr_pxy, rwvlr_pxy, logger, test_project)
 207
 208     def test_delete_descriptors(self, active_mgmt_session, test_project, logger):
 209         logger.info("Trying to delete the descriptors in project {}".format(test_project))
 210         rift.auto.descriptor.delete_descriptors(active_mgmt_session, test_project)
 211
 212     def test_delete_cloud_accounts(self, active_mgmt_session, logger, test_project, cloud_account):
 213         logger.info("Trying to delete the cloud-account in project {}".format(test_project))
 214         rift.auto.mano.delete_cloud_account(active_mgmt_session, cloud_account.name, test_project)
 215
 216     def test_delete_projects(self, active_mgmt_session, test_project, logger):
 217         rw_conman_proxy = active_mgmt_session.proxy(RwConmanYang)
 218         logger.debug('Deleting project {}'.format(test_project))
 219         rift.auto.mano.delete_project(rw_conman_proxy, test_project)