rwlaunchpad/ra/pytest/conftest.py

   1
   2 #
   3 #   Copyright 2016 RIFT.IO Inc
   4 #
   5 #   Licensed under the Apache License, Version 2.0 (the "License");
   6 #   you may not use this file except in compliance with the License.
   7 #   You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #   Unless required by applicable law or agreed to in writing, software
  12 #   distributed under the License is distributed on an "AS IS" BASIS,
  13 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #   See the License for the specific language governing permissions and
  15 #   limitations under the License.
  16 #
  17
  18 import gi
  19 import itertools
  20 import logging
  21 import os
  22 import pytest
  23 import random
  24 import re
  25 import rwlogger
  26 import rw_peas
  27 import subprocess
  28 import sys
  29
  30 import rift.auto.accounts
  31 import rift.auto.log
  32 import rift.auto.session
  33 import rift.rwcal.openstack
  34 import rift.vcs.vcs
  35
  36 from gi import require_version
  37 require_version('RwCloudYang', '1.0')
  38 require_version('RwTypes', '1.0')
  39 require_version('RwRbacPlatformYang', '1.0')
  40 require_version('RwUserYang', '1.0')
  41 require_version('RwProjectYang', '1.0')
  42 require_version('RwConmanYang', '1.0')
  43 require_version('RwRbacInternalYang', '1.0')
  44 require_version('RwRoAccountYang', '1.0')
  45
  46 from gi.repository import (
  47     RwCloudYang,
  48     RwTypes,
  49     RwUserYang,
  50     RwProjectYang,
  51     RwRbacPlatformYang,
  52     RwConmanYang,
  53     RwRbacInternalYang,
  54     RwRoAccountYang
  55 )
  56 gi.require_version('RwKeyspec', '1.0')
  57 from gi.repository.RwKeyspec import quoted_key
  58
  59 @pytest.fixture(scope='session')
  60 def use_accounts():
  61     account_names = os.environ.get('RW_AUTO_ACCOUNTS')
  62     if account_names:
  63         return account_names.split(":")
  64     return []
  65
  66 @pytest.fixture(scope='session')
  67 def account_storage():
  68     return rift.auto.accounts.Storage()
  69
  70 @pytest.fixture(scope='session')
  71 def stored_accounts(account_storage):
  72     return account_storage.list_cloud_accounts()
  73
  74 @pytest.fixture(scope='session')
  75 def cloud_name_prefix():
  76     '''fixture which returns the prefix used in cloud account names'''
  77     return 'cloud'
  78
  79 @pytest.fixture(scope='session')
  80 def cloud_account_name(cloud_account):
  81     '''fixture which returns the name used to identify the cloud account'''
  82     return cloud_account.name
  83
  84 @pytest.fixture(scope='session')
  85 def sdn_account_name():
  86     '''fixture which returns the name used to identify the sdn account'''
  87     return 'sdn-0'
  88
  89 @pytest.fixture(scope='session')
  90 def openstack_sdn_account_name():
  91     '''fixture which returns the name used to identify the sdn account'''
  92     return 'openstack-sdn-0'
  93
  94 @pytest.fixture(scope='session')
  95 def sdn_account_type():
  96     '''fixture which returns the account type used by the sdn account'''
  97     return 'odl'
  98
  99 @pytest.fixture(scope='session')
 100 def cloud_module():
 101     '''Fixture containing the module which defines cloud account
 102     Returns:
 103         module to be used when configuring a cloud account
 104     '''
 105     return RwCloudYang
 106
 107 @pytest.fixture(scope='session')
 108 def cloud_xpath():
 109     '''Fixture containing the xpath that should be used to configure a cloud account
 110     Returns:
 111         xpath to be used when configure a cloud account
 112     '''
 113     return '/rw-project:project[rw-project:name="default"]/cloud/account'
 114
 115 @pytest.fixture(scope='session')
 116 def cloud_accounts(request, cloud_module, cloud_name_prefix, cloud_host, cloud_user, cloud_tenants, cloud_type, stored_accounts, use_accounts, vim_host_override, vim_ssl_enabled, vim_user_domain_override, vim_project_domain_override, logger):
 117     '''fixture which returns a list of CloudAccounts. One per tenant provided
 118
 119     Arguments:
 120         cloud_module                - fixture: module defining cloud account
 121         cloud_name_prefix           - fixture: name prefix used for cloud account
 122         cloud_host                  - fixture: cloud host address
 123         cloud_user                  - fixture: cloud account user key
 124         cloud_tenants               - fixture: list of tenants to create cloud accounts on
 125         cloud_type                  - fixture: cloud account type
 126         stored_accounts             - fixture: account storage
 127         use_accounts                - fixture: use accounts from account storage
 128         vim_host_override           - fixture: use specified vim instead of account's vim
 129         vim_ssl_enabled             - fixture: enable or disable ssl regardless of accounts setting
 130         vim_user_domain_override    - fixture: use specified user domain instead of account's user domain
 131         vim_project_domain_override - fixture: use specified project domain instead of account's project domain
 132
 133     Returns:
 134         A list of CloudAccounts
 135     '''
 136
 137
 138     accounts = []
 139
 140     if use_accounts:
 141         for account_name in stored_accounts:
 142             if account_name in use_accounts:
 143                 if vim_host_override and stored_accounts[account_name].account_type == 'openstack':
 144                     old_auth = stored_accounts[account_name].openstack.auth_url
 145                     stored_accounts[account_name].openstack.auth_url = re.sub('(?:(?<=https://)|(?<=http://)).*?(?=:)', vim_host_override, old_auth)
 146                 if vim_ssl_enabled == False:
 147                     stored_accounts[account_name].openstack.auth_url = re.sub(
 148                         '^https',
 149                         'http',
 150                         stored_accounts[account_name].openstack.auth_url
 151                     )
 152                 elif vim_ssl_enabled == True:
 153                     stored_accounts[account_name].openstack.auth_url = re.sub(
 154                         '^http(?=:)',
 155                         'https',
 156                         stored_accounts[account_name].openstack.auth_url
 157                     )
 158                 if vim_user_domain_override:
 159                     stored_accounts[account_name].openstack.user_domain = vim_user_domain_override
 160                 if vim_project_domain_override:
 161                     stored_accounts[account_name].openstack.project_domain = vim_project_domain_override
 162                 accounts.append(stored_accounts[account_name])
 163     else:
 164         def account_name_generator(prefix):
 165             '''Generator of unique account names for a given prefix
 166             Arguments:
 167                 prefix - prefix of account name
 168             '''
 169             idx=0
 170             while True:
 171                 yield "{prefix}-{idx}".format(prefix=prefix, idx=idx)
 172                 idx+=1
 173         name_gen = account_name_generator(cloud_name_prefix)
 174
 175         for cloud_tenant in cloud_tenants:
 176             if cloud_type == 'lxc':
 177                 accounts.append(
 178                         cloud_module.CloudAcc.from_dict({
 179                             "name": next(name_gen),
 180                             "account_type": "cloudsim_proxy"})
 181                 )
 182             elif cloud_type == 'openstack':
 183                 hosts = [cloud_host]
 184                 if request.config.option.upload_images_multiple_accounts:
 185                     hosts.append('10.66.4.32')
 186                 for host in hosts:
 187                     password = 'mypasswd'
 188                     auth_url = 'http://{host}:5000/v3/'.format(host=host)
 189                     if vim_ssl_enabled == True:
 190                         auth_url = 'https://{host}:5000/v3/'.format(host=host)
 191                     mgmt_network = os.getenv('MGMT_NETWORK', 'private')
 192                     accounts.append(
 193                             cloud_module.YangData_RwProject_Project_Cloud_Account.from_dict({
 194                                 'name':  next(name_gen),
 195                                 'account_type': 'openstack',
 196                                 'openstack': {
 197                                     'admin': True,
 198                                     'key': cloud_user,
 199                                     'secret': password,
 200                                     'auth_url': auth_url,
 201                                     'tenant': cloud_tenant,
 202                                     'mgmt_network': mgmt_network,
 203                                     'floating_ip_pool': 'public',
 204                     }}))
 205             elif cloud_type == 'mock':
 206                 accounts.append(
 207                         cloud_module.CloudAcc.from_dict({
 208                             "name": next(name_gen),
 209                             "account_type": "mock"})
 210                 )
 211
 212     return accounts
 213
 214
 215 @pytest.fixture(scope='session', autouse=True)
 216 def cloud_account(cloud_accounts):
 217     '''fixture which returns an instance of RwCloudYang.CloudAcc
 218
 219     Arguments:
 220         cloud_accounts - fixture: list of generated cloud accounts
 221
 222     Returns:
 223         An instance of RwCloudYang.CloudAcc
 224     '''
 225     return cloud_accounts[0]
 226
 227 @pytest.fixture(scope='class')
 228 def vim_clients(cloud_accounts):
 229     """Fixture which returns sessions to VIMs"""
 230     vim_sessions = {}
 231     for cloud_account in cloud_accounts:
 232         if cloud_account.account_type == 'openstack':
 233             vim_sessions[cloud_account.name] = rift.rwcal.openstack.OpenstackDriver(**{
 234                 'username': cloud_account.openstack.key,
 235                 'password': cloud_account.openstack.secret,
 236                 'auth_url': cloud_account.openstack.auth_url,
 237                 'project':  cloud_account.openstack.tenant,
 238                 'mgmt_network': cloud_account.openstack.mgmt_network,
 239                 'cert_validate': cloud_account.openstack.cert_validate,
 240                 'user_domain': cloud_account.openstack.user_domain,
 241                 'project_domain': cloud_account.openstack.project_domain,
 242                 'region': cloud_account.openstack.region
 243             })
 244             # Add initialization for other VIM types
 245     return vim_sessions
 246
 247 @pytest.fixture(scope='session')
 248 def openmano_prefix():
 249     '''Fixture that returns the prefix to be used for openmano resource names'''
 250     return 'openmano'
 251
 252 @pytest.fixture(scope='session')
 253 def openmano_hosts(sut_host_names):
 254     '''Fixture that returns the set of host logical names to be used for openmano'''
 255     return [name for name in sut_host_names if 'openmano' in name]
 256
 257 @pytest.fixture(scope='session')
 258 def openmano_accounts(openmano_hosts, sut_host_addrs, cloud_accounts, openmano_prefix, logger):
 259     """Fixture that returns a list of Openmano accounts. One per host, and tenant provided"""
 260     accounts=[]
 261
 262     if not openmano_hosts:
 263         return accounts
 264
 265     host_cycle = itertools.cycle(openmano_hosts)
 266     for cloud_account in cloud_accounts:
 267         if cloud_account.account_type not in ['openstack']:
 268             logger.warning('Skipping creating ro datacenter for cloud account [%s] - unsupported account type [%s]', cloud_account.name, cloud_account.account_type)
 269             continue
 270
 271         try:
 272             host = next(host_cycle)
 273         except StopIteration:
 274             break
 275
 276         if cloud_account.account_type == 'openstack':
 277             accounts.append({
 278                 'account_name': "vim_%s" % cloud_account.name,
 279                 'openmano_tenant': host,
 280                 'openmano_addr': sut_host_addrs[host],
 281                 'openmano_port': 9090,
 282                 'datacenter': 'dc_%s' % (cloud_account.name),
 283                 'vim_account': cloud_account,
 284                 'vim_name': cloud_account.name,
 285                 'vim_type': cloud_account.account_type,
 286                 'vim_auth_url': cloud_account.openstack.auth_url,
 287                 'vim_user':cloud_account.openstack.key,
 288                 'vim_password':cloud_account.openstack.secret,
 289                 'vim_tenant':cloud_account.openstack.tenant,
 290             })
 291
 292     return accounts
 293
 294 @pytest.fixture(scope='session')
 295 def ro_account_info(openmano_accounts):
 296     ro_account_info = {}
 297     for account in openmano_accounts:
 298         ssh_cmd = (
 299             'ssh {openmano_addr} -q -n -o BatchMode=yes -o StrictHostKeyChecking=no -- '
 300         ).format(
 301             openmano_addr=account['openmano_addr']
 302         )
 303
 304         if account['account_name'] not in ro_account_info:
 305             tenant_create_cmd = (
 306                 '{ssh_cmd} openmano tenant-create {name}'
 307             ).format(
 308                 ssh_cmd=ssh_cmd,
 309                 name=account['account_name']
 310             )
 311             tenant_info = subprocess.check_output(tenant_create_cmd, shell=True).decode('ascii')
 312             (tenant_id, tenant_name) = tenant_info.split()
 313             ro_account_info[account['account_name']] = {
 314                 'tenant_id':tenant_id,
 315                 'account': account,
 316                 'account_type':'openmano',
 317                 'host':account['openmano_addr'],
 318                 'port':9090,
 319                 'datacenters':[],
 320             }
 321         else:
 322             tenant_id = ro_account_info[account['account_name']]['tenant_id']
 323
 324         datacenter_create_cmd = (
 325             '{ssh_cmd} openmano datacenter-create --type {vim_type} {datacenter} {vim_auth_url}'
 326         ).format(
 327             ssh_cmd=ssh_cmd,
 328             vim_type=account['vim_type'],
 329             datacenter=account['datacenter'],
 330             vim_auth_url=account['vim_auth_url']
 331         )
 332         datacenter_attach_cmd = (
 333                 '{ssh_cmd} OPENMANO_TENANT={tenant_id} openmano datacenter-attach {datacenter} --user={vim_user} '
 334                 '--password={vim_password} --vim-tenant-name={vim_tenant}'
 335         ).format(
 336             ssh_cmd=ssh_cmd,
 337             tenant_id=tenant_id,
 338             datacenter=account['datacenter'],
 339             vim_user=account['vim_user'],
 340             vim_password=account['vim_password'],
 341             vim_tenant=account['vim_tenant']
 342         )
 343         subprocess.check_call(datacenter_create_cmd, shell=True)
 344         subprocess.check_call(datacenter_attach_cmd, shell=True)
 345
 346         ro_account_info[account['account_name']]['datacenters'].append(account['datacenter'])
 347     return ro_account_info
 348
 349
 350 @pytest.fixture(scope='session')
 351 def ro_accounts(ro_account_info):
 352     '''Fixture that returns a map of RwRoAccountYang.ROAccount objects for each
 353     account in ro_account_info
 354     '''
 355     ro_accounts = {}
 356     for name, account_info in ro_account_info.items():
 357         ro_accounts[name] = RwRoAccountYang.YangData_RwProject_Project_RoAccount_Account.from_dict({
 358             'name':name,
 359             'ro_account_type':account_info['account_type'],
 360             'openmano':{
 361                 'host':account_info['host'],
 362                 'port':account_info['port'],
 363                 'tenant_id':account_info['tenant_id'],
 364             }
 365         })
 366     return ro_accounts
 367
 368 @pytest.fixture(scope='session')
 369 def ro_map(ro_account_info, ro_accounts):
 370     '''Fixture that returns a map of vim name to datacenter / ro name tuples for each account in ro_account_info
 371     '''
 372     ro_map = {}
 373     for account_name, account_info in ro_account_info.items():
 374         vim_name = account_info['account']['vim_account'].name
 375         datacenter_name = account_info['account']['datacenter']
 376         ro_map[vim_name] = (account_name, datacenter_name)
 377     return ro_map
 378
 379 @pytest.fixture(scope='session')
 380 def cal(cloud_account):
 381     """Fixture which returns cal interface"""
 382     if cloud_account.account_type == 'openstack':
 383         plugin = rw_peas.PeasPlugin('rwcal_openstack', 'RwCal-1.0')
 384     elif cloud_account.account_type == 'openvim':
 385         plugin = rw_peas.PeasPlugin('rwcal_openmano_vimconnector', 'RwCal-1.0')
 386     elif cloud_account.account_type == 'aws':
 387         plugin = rw_peas.PeasPlugin('rwcal_aws', 'RwCal-1.0')
 388     elif cloud_account.account_type == 'vsphere':
 389         plugin = rw_peas.PeasPlugin('rwcal-python', 'RwCal-1.0')
 390
 391     engine, info, extension = plugin()
 392     cal = plugin.get_interface("Cloud")
 393     rwloggerctx = rwlogger.RwLog.Ctx.new("Cal-Log")
 394     rc = cal.init(rwloggerctx)
 395     assert rc == RwTypes.RwStatus.SUCCESS
 396
 397     return cal
 398
 399 @pytest.fixture(scope='session')
 400 def rbac_user_passwd():
 401     """A common password being used for all rbac users."""
 402     return 'mypasswd'
 403
 404 @pytest.fixture(scope='session')
 405 def user_domain(tbac):
 406     """user-domain being used in this rbac test."""
 407     if tbac:
 408         return 'tbacdomain'
 409     return 'system'
 410
 411 @pytest.fixture(scope='session')
 412 def platform_roles():
 413     """Returns a tuple of platform roles"""
 414     return ('rw-rbac-platform:platform-admin', 'rw-rbac-platform:platform-oper', 'rw-rbac-platform:super-admin')
 415
 416 @pytest.fixture(scope='session')
 417 def user_roles():
 418     """Returns a tuple of user roles"""
 419     return ('rw-project:project-admin', 'rw-project:project-oper', 'rw-project-mano:catalog-oper', 'rw-project-mano:catalog-admin',
 420     'rw-project-mano:lcm-admin', 'rw-project-mano:lcm-oper', 'rw-project-mano:account-admin', 'rw-project-mano:account-oper',)
 421
 422 @pytest.fixture(scope='session')
 423 def all_roles(platform_roles, user_roles):
 424     """Returns a tuple of platform roles plus user roles"""
 425     return platform_roles + user_roles
 426
 427 @pytest.fixture(scope='session')
 428 def rw_user_proxy(mgmt_session):
 429     return mgmt_session.proxy(RwUserYang)
 430
 431 @pytest.fixture(scope='session')
 432 def rw_project_proxy(mgmt_session):
 433     return mgmt_session.proxy(RwProjectYang)
 434
 435 @pytest.fixture(scope='session')
 436 def rw_rbac_int_proxy(mgmt_session):
 437     return mgmt_session.proxy(RwRbacInternalYang)
 438
 439 @pytest.fixture(scope='session')
 440 def rw_ro_account_proxy(mgmt_session):
 441     return mgmt_session.proxy(RwRoAccountYang)
 442
 443 @pytest.fixture(scope='session')
 444 def rw_conman_proxy(mgmt_session):
 445     return mgmt_session.proxy(RwConmanYang)
 446
 447 @pytest.fixture(scope='session')
 448 def rbac_platform_proxy(mgmt_session):
 449     return mgmt_session.proxy(RwRbacPlatformYang)
 450
 451 @pytest.fixture(scope='session')
 452 def project_keyed_xpath():
 453     return '/project[name={project_name}]'
 454
 455 @pytest.fixture(scope='session')
 456 def user_keyed_xpath():
 457     return "/user-config/user[user-name={user}][user-domain={domain}]"
 458
 459 @pytest.fixture(scope='session')
 460 def platform_config_keyed_xpath():
 461     return "/rbac-platform-config/user[user-name={user}][user-domain={domain}]"
 462
 463 @pytest.fixture(scope='session')
 464 def fmt_vnfd_catalog_xpath():
 465     """Fixture that returns vnfd-catalog keyed xpath"""
 466     xpath = '/project[name={project}]/vnfd-catalog'
 467     return xpath
 468
 469 @pytest.fixture(scope='session')
 470 def fmt_vnfd_id_xpath():
 471     """Fixture that returns vnfd id xpath"""
 472     xpath = '/rw-project:project[rw-project:name={project}]/project-vnfd:vnfd-catalog/project-vnfd:vnfd[project-vnfd:id={vnfd_id}]'
 473     return xpath
 474
 475 @pytest.fixture(scope='session')
 476 def fmt_nsd_catalog_xpath():
 477     """Fixture that returns nsd-catalog keyed xpath"""
 478     xpath = '/project[name={project}]/nsd-catalog'
 479     return xpath
 480
 481 @pytest.fixture(scope='session')
 482 def fmt_nsd_id_xpath():
 483     """Fixture that returns nsd id xpath"""
 484     xpath = '/rw-project:project[rw-project:name={project}]/project-nsd:nsd-catalog/project-nsd:nsd[project-nsd:id={nsd_id}]'
 485     return xpath
 486
 487 @pytest.fixture(scope='session')
 488 def fmt_prefixed_cloud_xpath():
 489     """Fixture that returns cloud keyed xpath"""
 490     xpath = '/rw-project:project[rw-project:name={project}]/rw-cloud:cloud/rw-cloud:account[rw-cloud:name={account_name}]'
 491     return xpath
 492
 493 @pytest.fixture(scope='session')
 494 def fmt_cloud_xpath():
 495     """Fixture that returns cloud keyed xpath without yang prefix"""
 496     xpath = '/project[name={project}]/cloud/account[name={account_name}]'
 497     return xpath
 498
 499 @pytest.fixture(scope='session', autouse=True)
 500 def launchpad_glance_api_log():
 501     log_file = os.path.join(
 502         os.environ.get('HOME_RIFT', os.environ.get('RIFT_INSTALL')),
 503         'var','rift','log','glance','glance-api.log'
 504     )
 505     return log_file
 506
 507 @pytest.fixture(scope='session', autouse=True)
 508 def _glance_api_scraper_session(request, log_manager, confd_host, launchpad_glance_api_log):
 509     '''Fixture which returns an instance of rift.auto.log.FileSource to scrape
 510     the glance api logs of the launchpad host
 511     '''
 512     scraper = rift.auto.log.FileSource(host=confd_host, path=launchpad_glance_api_log)
 513     log_manager.source(source=scraper)
 514     return scraper