osm_nbi/tests/test.py

   1 #! /usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import getopt
  18 import sys
  19 import requests
  20 import json
  21 import logging
  22 import yaml
  23 # import json
  24 # import tarfile
  25 from time import sleep
  26 from random import randint
  27 import os
  28 from sys import stderr
  29
  30 __author__ = "Alfonso Tierno, alfonso.tiernosepulveda@telefonica.com"
  31 __date__ = "$2018-03-01$"
  32 __version__ = "0.3"
  33 version_date = "Oct 2018"
  34
  35
  36 def usage():
  37     print("Usage: ", sys.argv[0], "[options]")
  38     print("      Performs system tests over running NBI. It can be used for real OSM test using option '--test-osm'")
  39     print("      If this is the case env variables 'OSMNBITEST_VIM_NAME' must be supplied to create a VIM if not exist "
  40           "where deployment is done")
  41     print("OPTIONS")
  42     print("      -h|--help: shows this help")
  43     print("      --insecure: Allows non trusted https NBI server")
  44     print("      --list: list available tests")
  45     print("      --manual-check: Deployment tests stop after deployed to allow manual inspection. Only make sense with "
  46           "'--test-osm'")
  47     print("      -p|--password PASSWORD: NBI access password. 'admin' by default")
  48     print("      ---project PROJECT: NBI access project. 'admin' by default")
  49     print("      --test TEST[,...]: Execute only a test or a comma separated list of tests")
  50     print("      --params key=val: params to the previous test. key can be vnfd-files, nsd-file, ns-name, ns-config")
  51     print("      --test-osm: If missing this test is intended for NBI only, no other OSM components are expected. Use "
  52           "this flag to test the system. LCM and RO components are expected to be up and running")
  53     print("      --timeout TIMEOUT: General NBI timeout, by default {}s".format(timeout))
  54     print("      --timeout-deploy TIMEOUT: Timeout used for getting NS deployed, by default {}s".format(timeout_deploy))
  55     print("      --timeout-configure TIMEOUT: Timeout used for getting NS deployed and configured,"
  56           " by default {}s".format(timeout_configure))
  57     print("      -u|--user USERNAME: NBI access username. 'admin' by default")
  58     print("      --url URL: complete NBI server URL. 'https//localhost:9999/osm' by default")
  59     print("      -v|--verbose print debug information, can be used several times")
  60     print("      --no-verbose remove verbosity")
  61     print("      --version: prints current version")
  62     print("ENV variables used for real deployment tests with option osm-test.")
  63     print("      export OSMNBITEST_VIM_NAME=vim-name")
  64     print("      export OSMNBITEST_VIM_URL=vim-url")
  65     print("      export OSMNBITEST_VIM_TYPE=vim-type")
  66     print("      export OSMNBITEST_VIM_TENANT=vim-tenant")
  67     print("      export OSMNBITEST_VIM_USER=vim-user")
  68     print("      export OSMNBITEST_VIM_PASSWORD=vim-password")
  69     print("      export OSMNBITEST_VIM_CONFIG=\"vim-config\"")
  70     print("      export OSMNBITEST_NS_NAME=\"vim-config\"")
  71     return
  72
  73
  74 r_header_json = {"Content-type": "application/json"}
  75 headers_json = {"Content-type": "application/json", "Accept": "application/json"}
  76 r_header_yaml = {"Content-type": "application/yaml"}
  77 headers_yaml = {"Content-type": "application/yaml", "Accept": "application/yaml"}
  78 r_header_text = {"Content-type": "text/plain"}
  79 r_header_octect = {"Content-type": "application/octet-stream"}
  80 headers_text = {"Accept": "text/plain,application/yaml"}
  81 r_header_zip = {"Content-type": "application/zip"}
  82 headers_zip = {"Accept": "application/zip,application/yaml"}
  83 headers_zip_yaml = {"Accept": "application/yaml", "Content-type": "application/zip"}
  84 r_headers_yaml_location_vnfd = {"Location": "/vnfpkgm/v1/vnf_packages_content/", "Content-Type": "application/yaml"}
  85 r_headers_yaml_location_nsd = {"Location": "/nsd/v1/ns_descriptors_content/", "Content-Type": "application/yaml"}
  86 r_headers_yaml_location_nst = {"Location": "/nst/v1/netslice_templates_content", "Content-Type": "application/yaml"}
  87 r_headers_yaml_location_nslcmop = {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}
  88
  89 # test ones authorized
  90 test_authorized_list = (
  91     ("AU1", "Invalid vnfd id", "GET", "/vnfpkgm/v1/vnf_packages/non-existing-id",
  92      headers_json, None, 404, r_header_json, "json"),
  93     ("AU2", "Invalid nsd id", "GET", "/nsd/v1/ns_descriptors/non-existing-id",
  94      headers_yaml, None, 404, r_header_yaml, "yaml"),
  95     ("AU3", "Invalid nsd id", "DELETE", "/nsd/v1/ns_descriptors_content/non-existing-id",
  96      headers_yaml, None, 404, r_header_yaml, "yaml"),
  97 )
  98 timeout = 120   # general timeout
  99 timeout_deploy = 60*10        # timeout for NS deploying without charms
 100 timeout_configure = 60*20     # timeout for NS deploying and configuring
 101
 102
 103 class TestException(Exception):
 104     pass
 105
 106
 107 class TestRest:
 108     def __init__(self, url_base, header_base=None, verify=False, user="admin", password="admin", project="admin"):
 109         self.url_base = url_base
 110         if header_base is None:
 111             self.header_base = {}
 112         else:
 113             self.header_base = header_base.copy()
 114         self.s = requests.session()
 115         self.s.headers = self.header_base
 116         self.verify = verify
 117         self.token = False
 118         self.user = user
 119         self.password = password
 120         self.project = project
 121         self.vim_id = None
 122         # contains ID of tests obtained from Location response header. "" key contains last obtained id
 123         self.last_id = ""
 124         self.test_name = None
 125         self.step = 0   # number of subtest under test
 126         self.passed_tests = 0
 127         self.failed_tests = 0
 128
 129     def set_test_name(self, test_name):
 130         self.test_name = test_name
 131         self.step = 0
 132         self.last_id = ""
 133
 134     def set_header(self, header):
 135         self.s.headers.update(header)
 136
 137     def set_tet_name(self, test_name):
 138         self.test_name = test_name
 139
 140     def unset_header(self, key):
 141         if key in self.s.headers:
 142             del self.s.headers[key]
 143
 144     def test(self, description, method, url, headers, payload, expected_codes, expected_headers,
 145              expected_payload, store_file=None, pooling=False):
 146         """
 147         Performs an http request and check http code response. Exit if different than allowed. It get the returned id
 148         that can be used by following test in the URL with {name} where name is the name of the test
 149         :param description:  description of the test
 150         :param method: HTTP method: GET,PUT,POST,DELETE,...
 151         :param url: complete URL or relative URL
 152         :param headers: request headers to add to the base headers
 153         :param payload: Can be a dict, transformed to json, a text or a file if starts with '@'
 154         :param expected_codes: expected response codes, can be int, int tuple or int range
 155         :param expected_headers: expected response headers, dict with key values
 156         :param expected_payload: expected payload, 0 if empty, 'yaml', 'json', 'text', 'zip', 'octet-stream'
 157         :param store_file: filename to store content
 158         :param pooling: if True do not count neither log this test. Because a pooling is done with many equal requests
 159         :return: requests response
 160         """
 161         r = None
 162         try:
 163             if not self.s:
 164                 self.s = requests.session()
 165             # URL
 166             if not url:
 167                 url = self.url_base
 168             elif not url.startswith("http"):
 169                 url = self.url_base + url
 170
 171             # replace url <> with the last ID
 172             url = url.replace("<>", self.last_id)
 173             if payload:
 174                 if isinstance(payload, str):
 175                     if payload.startswith("@"):
 176                         mode = "r"
 177                         file_name = payload[1:]
 178                         if payload.startswith("@b"):
 179                             mode = "rb"
 180                             file_name = payload[2:]
 181                         with open(file_name, mode) as f:
 182                             payload = f.read()
 183                 elif isinstance(payload, dict):
 184                     payload = json.dumps(payload)
 185
 186             if not pooling:
 187                 test_description = "Test {}{} {} {} {}".format(self.test_name, self.step, description, method, url)
 188                 logger.warning(test_description)
 189                 self.step += 1
 190             stream = False
 191             if expected_payload in ("zip", "octet-string") or store_file:
 192                 stream = True
 193             __retry = 0
 194             while True:
 195                 try:
 196                     r = getattr(self.s, method.lower())(url, data=payload, headers=headers, verify=self.verify,
 197                                                         stream=stream)
 198                     break
 199                 except requests.exceptions.ConnectionError as e:
 200                     if __retry == 2:
 201                         raise
 202                     logger.error("Exception {}. Retrying".format(e))
 203                     __retry += 1
 204
 205             if expected_payload in ("zip", "octet-string") or store_file:
 206                 logger.debug("RX {}".format(r.status_code))
 207             else:
 208                 logger.debug("RX {}: {}".format(r.status_code, r.text))
 209
 210             # check response
 211             if expected_codes:
 212                 if isinstance(expected_codes, int):
 213                     expected_codes = (expected_codes,)
 214                 if r.status_code not in expected_codes:
 215                     raise TestException(
 216                         "Got status {}. Expected {}. {}".format(r.status_code, expected_codes, r.text))
 217
 218             if expected_headers:
 219                 for header_key, header_val in expected_headers.items():
 220                     if header_key.lower() not in r.headers:
 221                         raise TestException("Header {} not present".format(header_key))
 222                     if header_val and header_val.lower() not in r.headers[header_key]:
 223                         raise TestException("Header {} does not contain {} but {}".format(header_key, header_val,
 224                                             r.headers[header_key]))
 225
 226             if expected_payload is not None:
 227                 if expected_payload == 0 and len(r.content) > 0:
 228                     raise TestException("Expected empty payload")
 229                 elif expected_payload == "json":
 230                     try:
 231                         r.json()
 232                     except Exception as e:
 233                         raise TestException("Expected json response payload, but got Exception {}".format(e))
 234                 elif expected_payload == "yaml":
 235                     try:
 236                         yaml.safe_load(r.text)
 237                     except Exception as e:
 238                         raise TestException("Expected yaml response payload, but got Exception {}".format(e))
 239                 elif expected_payload in ("zip", "octet-string"):
 240                     if len(r.content) == 0:
 241                         raise TestException("Expected some response payload, but got empty")
 242                     # try:
 243                     #     tar = tarfile.open(None, 'r:gz', fileobj=r.raw)
 244                     #     for tarinfo in tar:
 245                     #         tarname = tarinfo.name
 246                     #         print(tarname)
 247                     # except Exception as e:
 248                     #     raise TestException("Expected zip response payload, but got Exception {}".format(e))
 249                 elif expected_payload == "text":
 250                     if len(r.content) == 0:
 251                         raise TestException("Expected some response payload, but got empty")
 252                     # r.text
 253             if store_file:
 254                 with open(store_file, 'wb') as fd:
 255                     for chunk in r.iter_content(chunk_size=128):
 256                         fd.write(chunk)
 257
 258             location = r.headers.get("Location")
 259             if location:
 260                 _id = location[location.rfind("/") + 1:]
 261                 if _id:
 262                     self.last_id = str(_id)
 263             if not pooling:
 264                 self.passed_tests += 1
 265             return r
 266         except TestException as e:
 267             self.failed_tests += 1
 268             r_status_code = None
 269             r_text = None
 270             if r:
 271                 r_status_code = r.status_code
 272                 r_text = r.text
 273             logger.error("{} \nRX code{}: {}".format(e, r_status_code, r_text))
 274             return None
 275             # exit(1)
 276         except IOError as e:
 277             if store_file:
 278                 logger.error("Cannot open file {}: {}".format(store_file, e))
 279             else:
 280                 logger.error("Exception: {}".format(e), exc_info=True)
 281             self.failed_tests += 1
 282             return None
 283             # exit(1)
 284         except requests.exceptions.RequestException as e:
 285             logger.error("Exception: {}".format(e))
 286
 287     def get_autorization(self):  # user=None, password=None, project=None):
 288         if self.token:  # and self.user == user and self.password == password and self.project == project:
 289             return
 290         # self.user = user
 291         # self.password = password
 292         # self.project = project
 293         r = self.test("Obtain token", "POST", "/admin/v1/tokens", headers_json,
 294                       {"username": self.user, "password": self.password, "project_id": self.project},
 295                       (200, 201), r_header_json, "json")
 296         if not r:
 297             return
 298         response = r.json()
 299         self.token = response["id"]
 300         self.set_header({"Authorization": "Bearer {}".format(self.token)})
 301
 302     def remove_authorization(self):
 303         if self.token:
 304             self.test("Delete token", "DELETE", "/admin/v1/tokens/{}".format(self.token), headers_json,
 305                       None, (200, 201, 204), None, None)
 306         self.token = None
 307         self.unset_header("Authorization")
 308
 309     def get_create_vim(self, test_osm):
 310         if self.vim_id:
 311             return self.vim_id
 312         self.get_autorization()
 313         if test_osm:
 314             vim_name = os.environ.get("OSMNBITEST_VIM_NAME")
 315             if not vim_name:
 316                 raise TestException(
 317                     "Needed to define OSMNBITEST_VIM_XXX variables to create a real VIM for deployment")
 318         else:
 319             vim_name = "fakeVim"
 320         # Get VIM
 321         r = self.test("Get VIM ID", "GET", "/admin/v1/vim_accounts?name={}".format(vim_name), headers_json,
 322                       None, 200, r_header_json, "json")
 323         if not r:
 324             return
 325         vims = r.json()
 326         if vims:
 327             return vims[0]["_id"]
 328         # Add VIM
 329         if test_osm:
 330             # check needed environ parameters:
 331             if not os.environ.get("OSMNBITEST_VIM_URL") or not os.environ.get("OSMNBITEST_VIM_TENANT"):
 332                 raise TestException("Env OSMNBITEST_VIM_URL and OSMNBITEST_VIM_TENANT are needed for create a real VIM"
 333                                     " to deploy on whit the --test-osm option")
 334             vim_data = "{{schema_version: '1.0', name: '{}', vim_type: {}, vim_url: '{}', vim_tenant_name: '{}', "\
 335                        "vim_user: {}, vim_password: {}".format(vim_name,
 336                                                                os.environ.get("OSMNBITEST_VIM_TYPE", "openstack"),
 337                                                                os.environ.get("OSMNBITEST_VIM_URL"),
 338                                                                os.environ.get("OSMNBITEST_VIM_TENANT"),
 339                                                                os.environ.get("OSMNBITEST_VIM_USER"),
 340                                                                os.environ.get("OSMNBITEST_VIM_PASSWORD"))
 341             if os.environ.get("OSMNBITEST_VIM_CONFIG"):
 342                 vim_data += " ,config: {}".format(os.environ.get("OSMNBITEST_VIM_CONFIG"))
 343             vim_data += "}"
 344         else:
 345             vim_data = "{schema_version: '1.0', name: fakeVim, vim_type: openstack, vim_url: 'http://10.11.12.13/fake'"\
 346                        ", vim_tenant_name: 'vimtenant', vim_user: vimuser, vim_password: vimpassword}"
 347         self.test("Create VIM", "POST", "/admin/v1/vim_accounts", headers_yaml, vim_data,
 348                   (201), {"Location": "/admin/v1/vim_accounts/", "Content-Type": "application/yaml"}, "yaml")
 349         return self.last_id
 350
 351     def print_results(self):
 352         print("\n\n\n--------------------------------------------")
 353         print("TEST RESULTS: Total: {}, Passed: {}, Failed: {}".format(self.passed_tests + self.failed_tests,
 354                                                                        self.passed_tests, self.failed_tests))
 355         print("--------------------------------------------")
 356
 357     def wait_until_delete(self, url_op, timeout_delete):
 358         """
 359         Make a pooling until topic is not present, because of deleted
 360         :param url_op:
 361         :param timeout_delete:
 362         :return:
 363         """
 364         description = "Wait to topic being deleted"
 365         test_description = "Test {}{} {} {} {}".format(self.test_name, self.step, description, "GET", url_op)
 366         logger.warning(test_description)
 367         self.step += 1
 368
 369         wait = timeout_delete
 370         while wait >= 0:
 371             r = self.test(description, "GET", url_op, headers_yaml, None, (200, 404), None, r_header_yaml, "yaml",
 372                           pooling=True)
 373             if not r:
 374                 return
 375             if r.status_code == 404:
 376                 self.passed_tests += 1
 377                 break
 378             elif r.status_code == 200:
 379                 wait -= 5
 380                 sleep(5)
 381         else:
 382             raise TestException("Topic is not deleted after {} seconds".format(timeout_delete))
 383             self.failed_tests += 1
 384
 385     def wait_operation_ready(self, ns_nsi, opp_id, timeout, expected_fail=False):
 386         """
 387         Wait until nslcmop or nsilcmop finished
 388         :param ns_nsi: "ns" o "nsi"
 389         :param opp_id: Id o fthe operation
 390         :param timeout:
 391         :param expected_fail:
 392         :return: None. Updates passed/failed_tests
 393         """
 394         if ns_nsi == "ns":
 395             url_op = "/nslcm/v1/ns_lcm_op_occs/{}".format(opp_id)
 396         else:
 397             url_op = "/nsilcm/v1/nsi_lcm_op_occs/{}".format(opp_id)
 398         description = "Wait to {} lcm operation complete".format(ns_nsi)
 399         test_description = "Test {}{} {} {} {}".format(self.test_name, self.step, description, "GET", url_op)
 400         logger.warning(test_description)
 401         self.step += 1
 402         wait = timeout
 403         while wait >= 0:
 404             r = self.test(description, "GET", url_op, headers_json, None,
 405                           200, r_header_json, "json", pooling=True)
 406             if not r:
 407                 return
 408             nslcmop = r.json()
 409             if "COMPLETED" in nslcmop["operationState"]:
 410                 if expected_fail:
 411                     logger.error("NS terminate has success, expecting failing: {}".format(nslcmop["detailed-status"]))
 412                     self.failed_tests += 1
 413                 else:
 414                     self.passed_tests += 1
 415                 break
 416             elif "FAILED" in nslcmop["operationState"]:
 417                 if not expected_fail:
 418                     logger.error("NS terminate has failed: {}".format(nslcmop["detailed-status"]))
 419                     self.failed_tests += 1
 420                 else:
 421                     self.passed_tests += 1
 422                 break
 423
 424             print(".", end="", file=stderr)
 425             wait -= 10
 426             sleep(10)
 427         else:
 428             self.failed_tests += 1
 429             logger.error("NS instantiate is not terminate after {} seconds".format(timeout))
 430             return
 431         print("", file=stderr)
 432
 433
 434 class TestNonAuthorized:
 435     description = "Test invalid URLs. methods and no authorization"
 436
 437     @staticmethod
 438     def run(engine, test_osm, manual_check, test_params=None):
 439         engine.set_test_name("NonAuth")
 440         engine.remove_authorization()
 441         test_not_authorized_list = (
 442             ("Invalid token", "GET", "/admin/v1/users", headers_json, None, 401, r_header_json, "json"),
 443             ("Invalid URL", "POST", "/admin/v1/nonexist", headers_yaml, None, 405, r_header_yaml, "yaml"),
 444             ("Invalid version", "DELETE", "/admin/v2/users", headers_yaml, None, 405, r_header_yaml, "yaml"),
 445         )
 446         for t in test_not_authorized_list:
 447             engine.test(*t)
 448
 449
 450 class TestUsersProjects:
 451     description = "test project and user creation"
 452
 453     @staticmethod
 454     def run(engine, test_osm, manual_check, test_params=None):
 455         engine.set_test_name("UserProject")
 456         engine.get_autorization()
 457         engine.test("Create project non admin", "POST", "/admin/v1/projects", headers_json, {"name": "P1"},
 458                     (201, 204), {"Location": "/admin/v1/projects/", "Content-Type": "application/json"}, "json")
 459         engine.test("Create project admin", "POST", "/admin/v1/projects", headers_json,
 460                     {"name": "Padmin", "admin": True}, (201, 204),
 461                     {"Location": "/admin/v1/projects/", "Content-Type": "application/json"}, "json")
 462         engine.test("Create project bad format", "POST", "/admin/v1/projects", headers_json, {"name": 1}, 422,
 463                     r_header_json, "json")
 464         engine.test("Create user with bad project", "POST", "/admin/v1/users", headers_json,
 465                     {"username": "U1", "projects": ["P1", "P2", "Padmin"], "password": "pw1"}, 409,
 466                     r_header_json, "json")
 467         engine.test("Create user with bad project and force", "POST", "/admin/v1/users?FORCE=True", headers_json,
 468                     {"username": "U1", "projects": ["P1", "P2", "Padmin"], "password": "pw1"}, 201,
 469                     {"Location": "/admin/v1/users/", "Content-Type": "application/json"}, "json")
 470         engine.test("Create user 2", "POST", "/admin/v1/users", headers_json,
 471                     {"username": "U2", "projects": ["P1"], "password": "pw2"}, 201,
 472                     {"Location": "/admin/v1/users/", "Content-Type": "application/json"}, "json")
 473         engine.test("Edit user U1, delete  P2 project", "PATCH", "/admin/v1/users/U1", headers_json,
 474                     {"projects": {"$'P2'": None}}, 204, None, None)
 475         res = engine.test("Check user U1, contains the right projects", "GET", "/admin/v1/users/U1",
 476                           headers_json, None, 200, None, json)
 477         if res:
 478             u1 = res.json()
 479             # print(u1)
 480             expected_projects = ["P1", "Padmin"]
 481             if u1["projects"] != expected_projects:
 482                 logger.error("User content projects '{}' different than expected '{}'. Edition has not done"
 483                              " properly".format(u1["projects"], expected_projects))
 484                 engine.failed_tests += 1
 485
 486         engine.test("Edit user U1, set Padmin as default project", "PUT", "/admin/v1/users/U1", headers_json,
 487                     {"projects": {"$'Padmin'": None, "$+[0]": "Padmin"}}, 204, None, None)
 488         res = engine.test("Check user U1, contains the right projects", "GET", "/admin/v1/users/U1",
 489                           headers_json, None, 200, None, json)
 490         if res:
 491             u1 = res.json()
 492             # print(u1)
 493             expected_projects = ["Padmin", "P1"]
 494             if u1["projects"] != expected_projects:
 495                 logger.error("User content projects '{}' different than expected '{}'. Edition has not done"
 496                              " properly".format(u1["projects"], expected_projects))
 497                 engine.failed_tests += 1
 498
 499         engine.test("Edit user U1, change password", "PATCH", "/admin/v1/users/U1", headers_json,
 500                     {"password": "pw1_new"}, 204, None, None)
 501
 502         engine.test("Change to project P1 non existing", "POST", "/admin/v1/tokens/", headers_json,
 503                     {"project_id": "P1"}, 401, r_header_json, "json")
 504
 505         res = engine.test("Change to user U1 project P1", "POST", "/admin/v1/tokens", headers_json,
 506                           {"username": "U1", "password": "pw1_new", "project_id": "P1"}, (200, 201),
 507                           r_header_json, "json")
 508         if res:
 509             response = res.json()
 510             engine.set_header({"Authorization": "Bearer {}".format(response["id"])})
 511
 512         engine.test("Edit user projects non admin", "PUT", "/admin/v1/users/U1", headers_json,
 513                     {"projects": {"$'P1'": None}}, 401, r_header_json, "json")
 514         engine.test("Add new project non admin", "POST", "/admin/v1/projects", headers_json,
 515                     {"name": "P2"}, 401, r_header_json, "json")
 516         engine.test("Add new user non admin", "POST", "/admin/v1/users", headers_json,
 517                     {"username": "U3", "projects": ["P1"], "password": "pw3"}, 401,
 518                     r_header_json, "json")
 519
 520         res = engine.test("Change to user U1 project Padmin", "POST", "/admin/v1/tokens", headers_json,
 521                           {"project_id": "Padmin"}, (200, 201), r_header_json, "json")
 522         if res:
 523             response = res.json()
 524             engine.set_header({"Authorization": "Bearer {}".format(response["id"])})
 525
 526         engine.test("Add new project admin", "POST", "/admin/v1/projects", headers_json, {"name": "P2"},
 527                     (201, 204), {"Location": "/admin/v1/projects/", "Content-Type": "application/json"}, "json")
 528         engine.test("Add new user U3 admin", "POST", "/admin/v1/users",
 529                     headers_json, {"username": "U3", "projects": ["P2"], "password": "pw3"}, (201, 204),
 530                     {"Location": "/admin/v1/users/", "Content-Type": "application/json"}, "json")
 531         engine.test("Edit user projects admin", "PUT", "/admin/v1/users/U3", headers_json,
 532                     {"projects": ["P2"]}, 204, None, None)
 533
 534         engine.test("Delete project P2 conflict", "DELETE", "/admin/v1/projects/P2", headers_json, None, 409,
 535                     r_header_json, "json")
 536         engine.test("Delete project P2 forcing", "DELETE", "/admin/v1/projects/P2?FORCE=True", headers_json,
 537                     None, 204, None, None)
 538
 539         engine.test("Delete user U1. Conflict deleting own user", "DELETE", "/admin/v1/users/U1", headers_json,
 540                     None, 409, r_header_json, "json")
 541         engine.test("Delete user U2", "DELETE", "/admin/v1/users/U2", headers_json, None, 204, None, None)
 542         engine.test("Delete user U3", "DELETE", "/admin/v1/users/U3", headers_json, None, 204, None, None)
 543         # change to admin
 544         engine.remove_authorization()   # To force get authorization
 545         engine.get_autorization()
 546         engine.test("Delete user U1", "DELETE", "/admin/v1/users/U1", headers_json, None, 204, None, None)
 547         engine.test("Delete project P1", "DELETE", "/admin/v1/projects/P1", headers_json, None, 204, None, None)
 548         engine.test("Delete project Padmin", "DELETE", "/admin/v1/projects/Padmin", headers_json, None, 204,
 549                     None, None)
 550
 551
 552 class TestFakeVim:
 553     description = "Creates/edit/delete fake VIMs and SDN controllers"
 554
 555     def __init__(self):
 556         self.vim = {
 557             "schema_version": "1.0",
 558             "schema_type": "No idea",
 559             "name": "myVim",
 560             "description": "Descriptor name",
 561             "vim_type": "openstack",
 562             "vim_url": "http://localhost:/vim",
 563             "vim_tenant_name": "vimTenant",
 564             "vim_user": "user",
 565             "vim_password": "password",
 566             "config": {"config_param": 1}
 567         }
 568         self.sdn = {
 569             "name": "sdn-name",
 570             "description": "sdn-description",
 571             "dpid": "50:50:52:54:00:94:21:21",
 572             "ip": "192.168.15.17",
 573             "port": 8080,
 574             "type": "opendaylight",
 575             "version": "3.5.6",
 576             "user": "user",
 577             "password": "passwd"
 578         }
 579         self.port_mapping = [
 580             {"compute_node": "compute node 1",
 581              "ports": [{"pci": "0000:81:00.0", "switch_port": "port-2/1", "switch_mac": "52:54:00:94:21:21"},
 582                        {"pci": "0000:81:00.1", "switch_port": "port-2/2", "switch_mac": "52:54:00:94:21:22"}
 583                        ]},
 584             {"compute_node": "compute node 2",
 585              "ports": [{"pci": "0000:81:00.0", "switch_port": "port-2/3", "switch_mac": "52:54:00:94:21:23"},
 586                        {"pci": "0000:81:00.1", "switch_port": "port-2/4", "switch_mac": "52:54:00:94:21:24"}
 587                        ]}
 588         ]
 589
 590     def run(self, engine, test_osm, manual_check, test_params=None):
 591
 592         vim_bad = self.vim.copy()
 593         vim_bad.pop("name")
 594
 595         engine.set_test_name("FakeVim")
 596         engine.get_autorization()
 597         engine.test("Create VIM", "POST", "/admin/v1/vim_accounts", headers_json, self.vim, (201, 204),
 598                     {"Location": "/admin/v1/vim_accounts/", "Content-Type": "application/json"}, "json")
 599         vim_id = engine.last_id
 600         engine.test("Create VIM without name, bad schema", "POST", "/admin/v1/vim_accounts", headers_json,
 601                     vim_bad, 422, None, headers_json)
 602         engine.test("Create VIM name repeated", "POST", "/admin/v1/vim_accounts", headers_json, self.vim,
 603                     409, None, headers_json)
 604         engine.test("Show VIMs", "GET", "/admin/v1/vim_accounts", headers_yaml, None, 200, r_header_yaml,
 605                     "yaml")
 606         engine.test("Show VIM", "GET", "/admin/v1/vim_accounts/{}".format(vim_id), headers_yaml, None, 200,
 607                     r_header_yaml, "yaml")
 608         if not test_osm:
 609             # delete with FORCE
 610             engine.test("Delete VIM", "DELETE", "/admin/v1/vim_accounts/{}?FORCE=True".format(vim_id), headers_yaml,
 611                         None, 202, None, 0)
 612             engine.test("Check VIM is deleted", "GET", "/admin/v1/vim_accounts/{}".format(vim_id), headers_yaml, None,
 613                         404, r_header_yaml, "yaml")
 614         else:
 615             # delete and wait until is really deleted
 616             engine.test("Delete VIM", "DELETE", "/admin/v1/vim_accounts/{}".format(vim_id), headers_yaml, None, 202,
 617                         None, 0)
 618             engine.wait_until_delete("/admin/v1/vim_accounts/{}".format(vim_id), timeout)
 619
 620
 621 class TestVIMSDN(TestFakeVim):
 622     description = "Creates VIM with SDN editing SDN controllers and port_mapping"
 623
 624     def __init__(self):
 625         TestFakeVim.__init__(self)
 626         self.wim = {
 627             "schema_version": "1.0",
 628             "schema_type": "No idea",
 629             "name": "myWim",
 630             "description": "Descriptor name",
 631             "wim_type": "odl",
 632             "wim_url": "http://localhost:/wim",
 633             "user": "user",
 634             "password": "password",
 635             "config": {"config_param": 1}
 636         }
 637
 638     def run(self, engine, test_osm, manual_check, test_params=None):
 639         engine.set_test_name("VimSdn")
 640         engine.get_autorization()
 641         # Added SDN
 642         engine.test("Create SDN", "POST", "/admin/v1/sdns", headers_json, self.sdn, (201, 204),
 643                     {"Location": "/admin/v1/sdns/", "Content-Type": "application/json"}, "json")
 644         sdnc_id = engine.last_id
 645         # sleep(5)
 646         # Edit SDN
 647         engine.test("Edit SDN", "PATCH", "/admin/v1/sdns/{}".format(sdnc_id), headers_json, {"name": "new_sdn_name"},
 648                     204, None, None)
 649         # sleep(5)
 650         # VIM with SDN
 651         self.vim["config"]["sdn-controller"] = sdnc_id
 652         self.vim["config"]["sdn-port-mapping"] = self.port_mapping
 653         engine.test("Create VIM", "POST", "/admin/v1/vim_accounts", headers_json, self.vim, (200, 204, 201),
 654                     {"Location": "/admin/v1/vim_accounts/", "Content-Type": "application/json"}, "json"),
 655
 656         vim_id = engine.last_id
 657         self.port_mapping[0]["compute_node"] = "compute node XX"
 658         engine.test("Edit VIM change port-mapping", "PUT", "/admin/v1/vim_accounts/{}".format(vim_id), headers_json,
 659                     {"config": {"sdn-port-mapping": self.port_mapping}}, 204, None, None)
 660         engine.test("Edit VIM remove port-mapping", "PUT", "/admin/v1/vim_accounts/{}".format(vim_id), headers_json,
 661                     {"config": {"sdn-port-mapping": None}}, 204, None, None)
 662
 663         engine.test("Create WIM", "POST", "/admin/v1/wim_accounts", headers_json, self.wim, (200, 204, 201),
 664                     {"Location": "/admin/v1/wim_accounts/", "Content-Type": "application/json"}, "json"),
 665         wim_id = engine.last_id
 666
 667         if not test_osm:
 668             # delete with FORCE
 669             engine.test("Delete VIM remove port-mapping", "DELETE",
 670                         "/admin/v1/vim_accounts/{}?FORCE=True".format(vim_id), headers_json, None, 202, None, 0)
 671             engine.test("Delete SDNC", "DELETE", "/admin/v1/sdns/{}?FORCE=True".format(sdnc_id), headers_json, None,
 672                         202, None, 0)
 673
 674             engine.test("Delete WIM", "DELETE",
 675                         "/admin/v1/wim_accounts/{}?FORCE=True".format(wim_id), headers_json, None, 202, None, 0)
 676             engine.test("Check VIM is deleted", "GET", "/admin/v1/vim_accounts/{}".format(vim_id), headers_yaml,
 677                         None, 404, r_header_yaml, "yaml")
 678             engine.test("Check SDN is deleted", "GET", "/admin/v1/sdns/{}".format(sdnc_id), headers_yaml, None,
 679                         404, r_header_yaml, "yaml")
 680             engine.test("Check WIM is deleted", "GET", "/admin/v1/wim_accounts/{}".format(wim_id), headers_yaml,
 681                         None, 404, r_header_yaml, "yaml")
 682         else:
 683             if manual_check:
 684                 input('VIM, SDN, WIM has been deployed. Perform manual check and press enter to resume')
 685             # delete and wait until is really deleted
 686             engine.test("Delete VIM remove port-mapping", "DELETE", "/admin/v1/vim_accounts/{}".format(vim_id),
 687                         headers_json, None, (202, 201, 204), None, 0)
 688             engine.test("Delete SDN", "DELETE", "/admin/v1/sdns/{}".format(sdnc_id), headers_json, None,
 689                         (202, 201, 204), None, 0)
 690             engine.test("Delete VIM", "DELETE", "/admin/v1/wim_accounts/{}".format(wim_id),
 691                         headers_json, None, (202, 201, 204), None, 0)
 692             engine.wait_until_delete("/admin/v1/vim_accounts/{}".format(vim_id), timeout)
 693             engine.wait_until_delete("/admin/v1/sdns/{}".format(sdnc_id), timeout)
 694             engine.wait_until_delete("/admin/v1/wim_accounts/{}".format(wim_id), timeout)
 695
 696
 697 class TestDeploy:
 698     description = "Base class for downloading descriptors from ETSI, onboard and deploy in real VIM"
 699
 700     def __init__(self):
 701         self.test_name = "DEPLOY"
 702         self.nsd_id = None
 703         self.vim_id = None
 704         self.ns_id = None
 705         self.vnfds_id = []
 706         self.descriptor_url = "https://osm-download.etsi.org/ftp/osm-3.0-three/2nd-hackfest/packages/"
 707         self.vnfd_filenames = ("cirros_vnf.tar.gz",)
 708         self.nsd_filename = "cirros_2vnf_ns.tar.gz"
 709         self.descriptor_edit = None
 710         self.uses_configuration = False
 711         self.users = {}
 712         self.passwords = {}
 713         self.commands = {}
 714         self.keys = {}
 715         self.timeout = 120
 716         self.qforce = ""
 717         self.ns_params = None
 718
 719     def create_descriptors(self, engine):
 720         temp_dir = os.path.dirname(os.path.abspath(__file__)) + "/temp/"
 721         if not os.path.exists(temp_dir):
 722             os.makedirs(temp_dir)
 723         for vnfd_index, vnfd_filename in enumerate(self.vnfd_filenames):
 724             if "/" in vnfd_filename:
 725                 vnfd_filename_path = vnfd_filename
 726                 if not os.path.exists(vnfd_filename_path):
 727                     raise TestException("File '{}' does not exist".format(vnfd_filename_path))
 728             else:
 729                 vnfd_filename_path = temp_dir + vnfd_filename
 730                 if not os.path.exists(vnfd_filename_path):
 731                     with open(vnfd_filename_path, "wb") as file:
 732                         response = requests.get(self.descriptor_url + vnfd_filename)
 733                         if response.status_code >= 300:
 734                             raise TestException("Error downloading descriptor from '{}': {}".format(
 735                                 self.descriptor_url + vnfd_filename, response.status_code))
 736                         file.write(response.content)
 737             if vnfd_filename_path.endswith(".yaml"):
 738                 headers = headers_yaml
 739             else:
 740                 headers = headers_zip_yaml
 741             if randint(0, 1) == 0:
 742                 # vnfd CREATE AND UPLOAD in one step:
 743                 engine.test("Onboard VNFD in one step", "POST",
 744                             "/vnfpkgm/v1/vnf_packages_content" + self.qforce, headers, "@b" + vnfd_filename_path, 201,
 745                             r_headers_yaml_location_vnfd,
 746                             "yaml")
 747                 self.vnfds_id.append(engine.last_id)
 748             else:
 749                 # vnfd CREATE AND UPLOAD ZIP
 750                 engine.test("Onboard VNFD step 1", "POST", "/vnfpkgm/v1/vnf_packages",
 751                             headers_json, None, 201,
 752                             {"Location": "/vnfpkgm/v1/vnf_packages/", "Content-Type": "application/json"}, "json")
 753                 self.vnfds_id.append(engine.last_id)
 754                 engine.test("Onboard VNFD step 2 as ZIP", "PUT",
 755                             "/vnfpkgm/v1/vnf_packages/<>/package_content" + self.qforce,
 756                             headers, "@b" + vnfd_filename_path, 204, None, 0)
 757
 758             if self.descriptor_edit:
 759                 if "vnfd{}".format(vnfd_index) in self.descriptor_edit:
 760                     # Modify VNFD
 761                     engine.test("Edit VNFD ", "PATCH",
 762                                 "/vnfpkgm/v1/vnf_packages/{}".format(self.vnfds_id[-1]),
 763                                 headers_yaml, self.descriptor_edit["vnfd{}".format(vnfd_index)], 204, None, None)
 764
 765         if "/" in self.nsd_filename:
 766             nsd_filename_path = self.nsd_filename
 767             if not os.path.exists(nsd_filename_path):
 768                 raise TestException("File '{}' does not exist".format(nsd_filename_path))
 769         else:
 770             nsd_filename_path = temp_dir + self.nsd_filename
 771             if not os.path.exists(nsd_filename_path):
 772                 with open(nsd_filename_path, "wb") as file:
 773                     response = requests.get(self.descriptor_url + self.nsd_filename)
 774                     if response.status_code >= 300:
 775                         raise TestException("Error downloading descriptor from '{}': {}".format(
 776                             self.descriptor_url + self.nsd_filename, response.status_code))
 777                     file.write(response.content)
 778         if nsd_filename_path.endswith(".yaml"):
 779             headers = headers_yaml
 780         else:
 781             headers = headers_zip_yaml
 782
 783         if randint(0, 1) == 0:
 784             # nsd CREATE AND UPLOAD in one step:
 785             engine.test("Onboard NSD in one step", "POST",
 786                         "/nsd/v1/ns_descriptors_content" + self.qforce, headers, "@b" + nsd_filename_path, 201,
 787                         r_headers_yaml_location_nsd, yaml)
 788             self.nsd_id = engine.last_id
 789         else:
 790             # nsd CREATE AND UPLOAD ZIP
 791             engine.test("Onboard NSD step 1", "POST", "/nsd/v1/ns_descriptors",
 792                         headers_json, None, 201,
 793                         {"Location": "/nsd/v1/ns_descriptors/", "Content-Type": "application/json"}, "json")
 794             self.nsd_id = engine.last_id
 795             engine.test("Onboard NSD step 2 as ZIP", "PUT",
 796                         "/nsd/v1/ns_descriptors/<>/nsd_content" + self.qforce,
 797                         headers, "@b" + nsd_filename_path, 204, None, 0)
 798
 799         if self.descriptor_edit and "nsd" in self.descriptor_edit:
 800             # Modify NSD
 801             engine.test("Edit NSD ", "PATCH",
 802                         "/nsd/v1/ns_descriptors/{}".format(self.nsd_id),
 803                         headers_yaml, self.descriptor_edit["nsd"], 204, None, None)
 804
 805     def delete_descriptors(self, engine):
 806         # delete descriptors
 807         engine.test("Delete NSSD SOL005", "DELETE",
 808                     "/nsd/v1/ns_descriptors/{}".format(self.nsd_id),
 809                     headers_yaml, None, 204, None, 0)
 810         for vnfd_id in self.vnfds_id:
 811             engine.test("Delete VNFD SOL005", "DELETE",
 812                         "/vnfpkgm/v1/vnf_packages/{}".format(vnfd_id), headers_yaml, None, 204, None, 0)
 813
 814     def instantiate(self, engine, ns_data):
 815         ns_data_text = yaml.safe_dump(ns_data, default_flow_style=True, width=256)
 816         # create NS Two steps
 817         r = engine.test("Create NS step 1", "POST", "/nslcm/v1/ns_instances",
 818                         headers_yaml, ns_data_text, 201,
 819                         {"Location": "nslcm/v1/ns_instances/", "Content-Type": "application/yaml"}, "yaml")
 820         if not r:
 821             return
 822         self.ns_id = engine.last_id
 823         engine.test("Instantiate NS step 2", "POST",
 824                     "/nslcm/v1/ns_instances/{}/instantiate".format(self.ns_id), headers_yaml, ns_data_text,
 825                     201, r_headers_yaml_location_nslcmop, "yaml")
 826         nslcmop_id = engine.last_id
 827
 828         if test_osm:
 829             # Wait until status is Ok
 830             timeout = timeout_configure if self.uses_configuration else timeout_deploy
 831             engine.wait_operation_ready("ns", nslcmop_id, timeout)
 832
 833     def terminate(self, engine):
 834         # remove deployment
 835         if test_osm:
 836             engine.test("Terminate NS", "POST", "/nslcm/v1/ns_instances/{}/terminate".format(self.ns_id), headers_yaml,
 837                         None, 201, r_headers_yaml_location_nslcmop, "yaml")
 838             nslcmop2_id = engine.last_id
 839             # Wait until status is Ok
 840             engine.wait_operation_ready("ns", nslcmop2_id, timeout_deploy)
 841
 842             engine.test("Delete NS", "DELETE", "/nslcm/v1/ns_instances/{}".format(self.ns_id), headers_yaml, None,
 843                         204, None, 0)
 844         else:
 845             engine.test("Delete NS with FORCE", "DELETE", "/nslcm/v1/ns_instances/{}?FORCE=True".format(self.ns_id),
 846                         headers_yaml, None, 204, None, 0)
 847
 848         # check all it is deleted
 849         engine.test("Check NS is deleted", "GET", "/nslcm/v1/ns_instances/{}".format(self.ns_id), headers_yaml, None,
 850                     404, None, "yaml")
 851         r = engine.test("Check NSLCMOPs are deleted", "GET",
 852                         "/nslcm/v1/ns_lcm_op_occs?nsInstanceId={}".format(self.ns_id), headers_json, None,
 853                         200, None, "json")
 854         if not r:
 855             return
 856         nslcmops = r.json()
 857         if not isinstance(nslcmops, list) or nslcmops:
 858             raise TestException("NS {} deleted but with ns_lcm_op_occ active: {}".format(self.ns_id, nslcmops))
 859
 860     def test_ns(self, engine, test_osm, commands=None, users=None, passwds=None, keys=None, timeout=0):
 861
 862         r = engine.test("GET VNFR IDs", "GET",
 863                         "/nslcm/v1/ns_instances/{}".format(self.ns_id), headers_json, None,
 864                         200, r_header_json, "json")
 865         if not r:
 866             return
 867         ns_data = r.json()
 868
 869         vnfr_list = ns_data['constituent-vnfr-ref']
 870         time = 0
 871         _commands = commands if commands is not None else self.commands
 872         _users = users if users is not None else self.users
 873         _passwds = passwds if passwds is not None else self.passwords
 874         _keys = keys if keys is not None else self.keys
 875         _timeout = timeout if timeout != 0 else self.timeout
 876
 877         for vnfr_id in vnfr_list:
 878             r = engine.test("Get VNFR to get IP_ADDRESS", "GET",
 879                             "/nslcm/v1/vnfrs/{}".format(vnfr_id), headers_json, None,
 880                             200, r_header_json, "json")
 881             if not r:
 882                 continue
 883             vnfr_data = r.json()
 884
 885             vnf_index = str(vnfr_data["member-vnf-index-ref"])
 886             if not _commands.get(vnf_index):
 887                 continue
 888             if vnfr_data.get("ip-address"):
 889                 description = "Exec command='{}' at VNFR={} IP={}".format(_commands.get(vnf_index)[0], vnf_index,
 890                                                                           vnfr_data['ip-address'])
 891                 engine.step += 1
 892                 test_description = "{}{} {}".format(engine.test_name, engine.step, description)
 893                 logger.warning(test_description)
 894                 while _timeout >= time:
 895                     result, message = self.do_checks([vnfr_data["ip-address"]],
 896                                                      vnf_index=vnfr_data["member-vnf-index-ref"],
 897                                                      commands=_commands.get(vnf_index), user=_users.get(vnf_index),
 898                                                      passwd=_passwds.get(vnf_index), key=_keys.get(vnf_index))
 899                     if result == 1:
 900                         engine.passed_tests += 1
 901                         logger.debug(message)
 902                         break
 903                     elif result == 0:
 904                         time += 20
 905                         sleep(20)
 906                     elif result == -1:
 907                         engine.failed_tests += 1
 908                         logger.error(message)
 909                         break
 910                 else:
 911                     time -= 20
 912                     engine.failed_tests += 1
 913                     logger.error(message)
 914             else:
 915                 engine.failed_tests += 1
 916                 logger.error("VNFR {} has not mgmt address. Check failed".format(vnfr_id))
 917
 918     def do_checks(self, ip, vnf_index, commands=[], user=None, passwd=None, key=None):
 919         try:
 920             import urllib3
 921             from pssh.clients import ParallelSSHClient
 922             from pssh.utils import load_private_key
 923             from ssh2 import exceptions as ssh2Exception
 924         except ImportError as e:
 925             logger.critical("Package <pssh> or/and <urllib3> is not installed. Please add them with 'pip3 install "
 926                             "parallel-ssh urllib3': {}".format(e))
 927             return -1, "install needed packages 'pip3 install parallel-ssh urllib3'"
 928         urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 929         try:
 930             p_host = os.environ.get("PROXY_HOST")
 931             p_user = os.environ.get("PROXY_USER")
 932             p_password = os.environ.get("PROXY_PASSWD")
 933
 934             if key:
 935                 pkey = load_private_key(key)
 936             else:
 937                 pkey = None
 938
 939             client = ParallelSSHClient(ip, user=user, password=passwd, pkey=pkey, proxy_host=p_host,
 940                                        proxy_user=p_user, proxy_password=p_password, timeout=10, num_retries=0)
 941             for cmd in commands:
 942                 output = client.run_command(cmd)
 943                 client.join(output)
 944                 if output[ip[0]].exit_code:
 945                     return -1, "VNFR {} command '{}' returns error: '{}'".format(ip[0], cmd,
 946                                                                                  "\n".join(output[ip[0]].stderr))
 947                 else:
 948                     return 1, "VNFR {} command '{}' successful".format(ip[0], cmd)
 949         except (ssh2Exception.ChannelFailure, ssh2Exception.SocketDisconnectError, ssh2Exception.SocketTimeout,
 950                 ssh2Exception.SocketRecvError) as e:
 951             return 0, "Timeout accessing the VNFR {}: {}".format(ip[0], str(e))
 952         except Exception as e:
 953             return -1, "ERROR checking the VNFR {}: {}".format(ip[0], str(e))
 954
 955     def additional_operations(self, engine, test_osm, manual_check):
 956         pass
 957
 958     def run(self, engine, test_osm, manual_check, test_params=None):
 959         engine.set_test_name(self.test_name)
 960         engine.get_autorization()
 961         nsname = os.environ.get("OSMNBITEST_NS_NAME", "OSMNBITEST")
 962         if test_params:
 963             if "vnfd-files" in test_params:
 964                 self.vnfd_filenames = test_params["vnfd-files"].split(",")
 965             if "nsd-file" in test_params:
 966                 self.nsd_filename = test_params["nsd-file"]
 967             if test_params.get("ns-name"):
 968                 nsname = test_params["ns-name"]
 969         self.create_descriptors(engine)
 970
 971         # create real VIM if not exist
 972         self.vim_id = engine.get_create_vim(test_osm)
 973         ns_data = {"nsDescription": "default description", "nsName": nsname, "nsdId": self.nsd_id,
 974                    "vimAccountId": self.vim_id}
 975         if self.ns_params:
 976             ns_data.update(self.ns_params)
 977         if test_params and test_params.get("ns-config"):
 978             if isinstance(test_params["ns-config"], str):
 979                 ns_data.update(yaml.load(test_params["ns-config"]))
 980             else:
 981                 ns_data.update(test_params["ns-config"])
 982         self.instantiate(engine, ns_data)
 983
 984         if manual_check:
 985             input('NS has been deployed. Perform manual check and press enter to resume')
 986         if test_osm and self.commands:
 987             self.test_ns(engine, test_osm)
 988         self.additional_operations(engine, test_osm, manual_check)
 989         self.terminate(engine)
 990         self.delete_descriptors(engine)
 991
 992
 993 class TestDeployHackfestCirros(TestDeploy):
 994     description = "Load and deploy Hackfest cirros_2vnf_ns example"
 995
 996     def __init__(self):
 997         super().__init__()
 998         self.test_name = "CIRROS"
 999         self.vnfd_filenames = ("cirros_vnf.tar.gz",)
1000         self.nsd_filename = "cirros_2vnf_ns.tar.gz"
1001         self.commands = {'1': ['ls -lrt', ], '2': ['ls -lrt', ]}
1002         self.users = {'1': "cirros", '2': "cirros"}
1003         self.passwords = {'1': "cubswin:)", '2': "cubswin:)"}
1004
1005
1006 class TestDeployHackfest1(TestDeploy):
1007     description = "Load and deploy Hackfest_1_vnfd example"
1008
1009     def __init__(self):
1010         super().__init__()
1011         self.test_name = "HACKFEST1-"
1012         self.vnfd_filenames = ("hackfest_1_vnfd.tar.gz",)
1013         self.nsd_filename = "hackfest_1_nsd.tar.gz"
1014         # self.commands = {'1': ['ls -lrt', ], '2': ['ls -lrt', ]}
1015         # self.users = {'1': "cirros", '2': "cirros"}
1016         # self.passwords = {'1': "cubswin:)", '2': "cubswin:)"}
1017
1018
1019 class TestDeployHackfestCirrosScaling(TestDeploy):
1020     description = "Load and deploy Hackfest cirros_2vnf_ns example with scaling modifications"
1021
1022     def __init__(self):
1023         super().__init__()
1024         self.test_name = "CIRROS-SCALE"
1025         self.vnfd_filenames = ("cirros_vnf.tar.gz",)
1026         self.nsd_filename = "cirros_2vnf_ns.tar.gz"
1027
1028     def create_descriptors(self, engine):
1029         super().create_descriptors(engine)
1030         # Modify VNFD to add scaling and count=2
1031         self.descriptor_edit = {
1032             "vnfd0": {
1033                 "vdu": {
1034                     "$id: 'cirros_vnfd-VM'": {"count": 2}
1035                 },
1036                 "scaling-group-descriptor": [{
1037                     "name": "scale_cirros",
1038                     "max-instance-count": 2,
1039                     "vdu": [{
1040                         "vdu-id-ref": "cirros_vnfd-VM",
1041                         "count": 2
1042                     }]
1043                 }]
1044             }
1045         }
1046
1047     def additional_operations(self, engine, test_osm, manual_check):
1048         if not test_osm:
1049             return
1050         # 2 perform scale out twice
1051         payload = '{scaleType: SCALE_VNF, scaleVnfData: {scaleVnfType: SCALE_OUT, scaleByStepData: ' \
1052                   '{scaling-group-descriptor: scale_cirros, member-vnf-index: "1"}}}'
1053         for i in range(0, 2):
1054             engine.test("Execute scale action over NS", "POST",
1055                         "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload,
1056                         201, r_headers_yaml_location_nslcmop, "yaml")
1057             nslcmop2_scale_out = engine.last_id
1058             engine.wait_operation_ready("ns", nslcmop2_scale_out, timeout_deploy)
1059             if manual_check:
1060                 input('NS scale out done. Check that two more vdus are there')
1061             # TODO check automatic
1062
1063         # 2 perform scale in
1064         payload = '{scaleType: SCALE_VNF, scaleVnfData: {scaleVnfType: SCALE_IN, scaleByStepData: ' \
1065                   '{scaling-group-descriptor: scale_cirros, member-vnf-index: "1"}}}'
1066         for i in range(0, 2):
1067             engine.test("Execute scale IN action over NS", "POST",
1068                         "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload,
1069                         201, r_headers_yaml_location_nslcmop, "yaml")
1070             nslcmop2_scale_in = engine.last_id
1071             engine.wait_operation_ready("ns", nslcmop2_scale_in, timeout_deploy)
1072             if manual_check:
1073                 input('NS scale in done. Check that two less vdus are there')
1074             # TODO check automatic
1075
1076         # perform scale in that must fail as reached limit
1077         engine.test("Execute scale IN out of limit action over NS", "POST",
1078                     "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload,
1079                     201, r_headers_yaml_location_nslcmop, "yaml")
1080         nslcmop2_scale_in = engine.last_id
1081         engine.wait_operation_ready("ns", nslcmop2_scale_in, timeout_deploy, expected_fail=True)
1082
1083
1084 class TestDeployIpMac(TestDeploy):
1085     description = "Load and deploy descriptor examples setting mac, ip address at descriptor and instantiate params"
1086
1087     def __init__(self):
1088         super().__init__()
1089         self.test_name = "SetIpMac"
1090         self.vnfd_filenames = ("vnfd_2vdu_set_ip_mac2.yaml", "vnfd_2vdu_set_ip_mac.yaml")
1091         self.nsd_filename = "scenario_2vdu_set_ip_mac.yaml"
1092         self.descriptor_url = \
1093             "https://osm.etsi.org/gitweb/?p=osm/RO.git;a=blob_plain;f=test/RO_tests/v3_2vdu_set_ip_mac/"
1094         self.commands = {'1': ['ls -lrt', ], '2': ['ls -lrt', ]}
1095         self.users = {'1': "osm", '2': "osm"}
1096         self.passwords = {'1': "osm4u", '2': "osm4u"}
1097         self.timeout = 360
1098
1099     def run(self, engine, test_osm, manual_check, test_params=None):
1100         # super().run(engine, test_osm, manual_check, test_params)
1101         # run again setting IPs with instantiate parameters
1102         instantiation_params = {
1103             "vnf": [
1104                 {
1105                     "member-vnf-index": "1",
1106                     "internal-vld": [
1107                         {
1108                             "name": "internal_vld1",   # net_internal
1109                             "ip-profile": {
1110                                 "ip-version": "ipv4",
1111                                 "subnet-address": "10.9.8.0/24",
1112                                 "dhcp-params": {"count": 100, "start-address": "10.9.8.100"}
1113                             },
1114                             "internal-connection-point": [
1115                                 {
1116                                     "id-ref": "eth2",
1117                                     "ip-address": "10.9.8.2",
1118                                 },
1119                                 {
1120                                     "id-ref": "eth3",
1121                                     "ip-address": "10.9.8.3",
1122                                 }
1123                             ]
1124                         },
1125                     ],
1126
1127                     "vdu": [
1128                         {
1129                             "id": "VM1",
1130                             "interface": [
1131                                 # {
1132                                 #     "name": "iface11",
1133                                 #     "floating-ip-required": True,
1134                                 # },
1135                                 {
1136                                     "name": "iface13",
1137                                     "mac-address": "52:33:44:55:66:13"
1138                                 },
1139                             ],
1140                         },
1141                         {
1142                             "id": "VM2",
1143                             "interface": [
1144                                 {
1145                                     "name": "iface21",
1146                                     "ip-address": "10.31.31.22",
1147                                     "mac-address": "52:33:44:55:66:21"
1148                                 },
1149                             ],
1150                         },
1151                     ]
1152                 },
1153             ]
1154         }
1155
1156         super().run(engine, test_osm, manual_check, test_params={"ns-config": instantiation_params})
1157
1158
1159 class TestDeployHackfest4(TestDeploy):
1160     description = "Load and deploy Hackfest 4 example."
1161
1162     def __init__(self):
1163         super().__init__()
1164         self.test_name = "HACKFEST4-"
1165         self.vnfd_filenames = ("hackfest_4_vnfd.tar.gz",)
1166         self.nsd_filename = "hackfest_4_nsd.tar.gz"
1167         self.uses_configuration = True
1168         self.commands = {'1': ['ls -lrt', ], '2': ['ls -lrt', ]}
1169         self.users = {'1': "ubuntu", '2': "ubuntu"}
1170         self.passwords = {'1': "osm4u", '2': "osm4u"}
1171
1172     def create_descriptors(self, engine):
1173         super().create_descriptors(engine)
1174         # Modify VNFD to add scaling
1175         self.descriptor_edit = {
1176             "vnfd0": {
1177                 'vnf-configuration': {
1178                     'config-primitive': [{
1179                         'name': 'touch',
1180                         'parameter': [{
1181                             'name': 'filename',
1182                             'data-type': 'STRING',
1183                             'default-value': '/home/ubuntu/touched'
1184                         }]
1185                     }]
1186                 },
1187                 'scaling-group-descriptor': [{
1188                     'name': 'scale_dataVM',
1189                     'scaling-policy': [{
1190                         'threshold-time': 0,
1191                         'name': 'auto_cpu_util_above_threshold',
1192                         'scaling-type': 'automatic',
1193                         'scaling-criteria': [{
1194                             'name': 'cpu_util_above_threshold',
1195                             'vnf-monitoring-param-ref': 'all_aaa_cpu_util',
1196                             'scale-out-relational-operation': 'GE',
1197                             'scale-in-threshold': 15,
1198                             'scale-out-threshold': 60,
1199                             'scale-in-relational-operation': 'LE'
1200                         }],
1201                         'cooldown-time': 60
1202                     }],
1203                     'max-instance-count': 10,
1204                     'scaling-config-action': [
1205                         {'vnf-config-primitive-name-ref': 'touch',
1206                          'trigger': 'post-scale-out'},
1207                         {'vnf-config-primitive-name-ref': 'touch',
1208                          'trigger': 'pre-scale-in'}
1209                     ],
1210                     'vdu': [{
1211                         'vdu-id-ref': 'dataVM',
1212                         'count': 1
1213                     }]
1214                 }]
1215             }
1216         }
1217
1218
1219 class TestDeployHackfest3Charmed(TestDeploy):
1220     description = "Load and deploy Hackfest 3charmed_ns example"
1221
1222     def __init__(self):
1223         super().__init__()
1224         self.test_name = "HACKFEST3-"
1225         self.vnfd_filenames = ("hackfest_3charmed_vnfd.tar.gz",)
1226         self.nsd_filename = "hackfest_3charmed_nsd.tar.gz"
1227         self.uses_configuration = True
1228         self.commands = {'1': ['ls -lrt /home/ubuntu/first-touch'], '2': ['ls -lrt /home/ubuntu/first-touch']}
1229         self.users = {'1': "ubuntu", '2': "ubuntu"}
1230         self.passwords = {'1': "osm4u", '2': "osm4u"}
1231
1232     def additional_operations(self, engine, test_osm, manual_check):
1233         if not test_osm:
1234             return
1235         # 1 perform action
1236         payload = '{member_vnf_index: "2", primitive: touch, primitive_params: { filename: /home/ubuntu/OSMTESTNBI }}'
1237         engine.test("Exec service primitive over NS", "POST",
1238                     "/nslcm/v1/ns_instances/{}/action".format(self.ns_id), headers_yaml, payload,
1239                     201, r_headers_yaml_location_nslcmop, "yaml")
1240         nslcmop2_action = engine.last_id
1241         # Wait until status is Ok
1242         engine.wait_operation_ready("ns", nslcmop2_action, timeout_deploy)
1243         if manual_check:
1244             input('NS service primitive has been executed. Check that file /home/ubuntu/OSMTESTNBI is present at '
1245                   'TODO_PUT_IP')
1246         if test_osm:
1247             commands = {'1': [''], '2': ['ls -lrt /home/ubuntu/OSMTESTNBI', ]}
1248             self.test_ns(engine, test_osm, commands=commands)
1249
1250         # # 2 perform scale out
1251         # payload = '{scaleType: SCALE_VNF, scaleVnfData: {scaleVnfType: SCALE_OUT, scaleByStepData: ' \
1252         #           '{scaling-group-descriptor: scale_dataVM, member-vnf-index: "1"}}}'
1253         # engine.test("Execute scale action over NS", "POST",
1254         #             "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload,
1255         #             201, r_headers_yaml_location_nslcmop, "yaml")
1256         # nslcmop2_scale_out = engine.last_id
1257         # engine.wait_operation_ready("ns", nslcmop2_scale_out, timeout_deploy)
1258         # if manual_check:
1259         #     input('NS scale out done. Check that file /home/ubuntu/touched is present and new VM is created')
1260         # # TODO check automatic
1261         #
1262         # # 2 perform scale in
1263         # payload = '{scaleType: SCALE_VNF, scaleVnfData: {scaleVnfType: SCALE_IN, scaleByStepData: ' \
1264         #           '{scaling-group-descriptor: scale_dataVM, member-vnf-index: "1"}}}'
1265         # engine.test("Execute scale action over NS", "POST",
1266         #             "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload,
1267         #             201, r_headers_yaml_location_nslcmop, "yaml")
1268         # nslcmop2_scale_in = engine.last_id
1269         # engine.wait_operation_ready("ns", nslcmop2_scale_in, timeout_deploy)
1270         # if manual_check:
1271         #     input('NS scale in done. Check that file /home/ubuntu/touched is updated and new VM is deleted')
1272         # # TODO check automatic
1273
1274
1275 class TestDeployHackfest3Charmed2(TestDeployHackfest3Charmed):
1276     description = "Load and deploy Hackfest 3charmed_ns example modified version of descriptors to have dots in " \
1277                   "ids and member-vnf-index."
1278
1279     def __init__(self):
1280         super().__init__()
1281         self.test_name = "HACKFEST3v2-"
1282         self.qforce = "?FORCE=True"
1283         self.descriptor_edit = {
1284             "vnfd0": {
1285                 "vdu": {
1286                     "$[0]": {
1287                         "interface": {"$[0]": {"external-connection-point-ref": "pdu-mgmt"}}
1288                     },
1289                     "$[1]": None
1290                 },
1291                 "vnf-configuration": None,
1292                 "connection-point": {
1293                     "$[0]": {
1294                         "id": "pdu-mgmt",
1295                         "name": "pdu-mgmt",
1296                         "short-name": "pdu-mgmt"
1297                     },
1298                     "$[1]": None
1299                 },
1300                 "mgmt-interface": {"cp": "pdu-mgmt"},
1301                 "description": "A vnf single vdu to be used as PDU",
1302                 "id": "vdu-as-pdu",
1303                 "internal-vld": {
1304                     "$[0]": {
1305                         "id": "pdu_internal",
1306                         "name": "pdu_internal",
1307                         "internal-connection-point": {"$[1]": None},
1308                         "short-name": "pdu_internal",
1309                         "type": "ELAN"
1310                     }
1311                 }
1312             },
1313
1314             # Modify NSD accordingly
1315             "nsd": {
1316                 "constituent-vnfd": {
1317                     "$[0]": {"vnfd-id-ref": "vdu-as-pdu"},
1318                     "$[1]": None,
1319                 },
1320                 "description": "A nsd to deploy the vnf to act as as PDU",
1321                 "id": "nsd-as-pdu",
1322                 "name": "nsd-as-pdu",
1323                 "short-name": "nsd-as-pdu",
1324                 "vld": {
1325                     "$[0]": {
1326                         "id": "mgmt_pdu",
1327                         "name": "mgmt_pdu",
1328                         "short-name": "mgmt_pdu",
1329                         "vnfd-connection-point-ref": {
1330                             "$[0]": {
1331                                 "vnfd-connection-point-ref": "pdu-mgmt",
1332                                 "vnfd-id-ref": "vdu-as-pdu",
1333                             },
1334                             "$[1]": None
1335                         },
1336                         "type": "ELAN"
1337                     },
1338                     "$[1]": None,
1339                 }
1340             }
1341         }
1342
1343
1344 class TestDeployHackfest3Charmed3(TestDeployHackfest3Charmed):
1345     description = "Load and deploy Hackfest 3charmed_ns example modified version to test scaling and NS parameters"
1346
1347     def __init__(self):
1348         super().__init__()
1349         self.test_name = "HACKFEST3v3-"
1350         self.commands = {'1': ['ls -lrt /home/ubuntu/first-touch-1'], '2': ['ls -lrt /home/ubuntu/first-touch-1']}
1351         self.descriptor_edit = {
1352             "vnfd0": yaml.load(
1353                 """
1354                 scaling-group-descriptor:
1355                     -   name: "scale_dataVM"
1356                         max-instance-count: 10
1357                         scaling-policy:
1358                         -   name: "auto_cpu_util_above_threshold"
1359                             scaling-type: "automatic"
1360                             threshold-time: 0
1361                             cooldown-time: 60
1362                             scaling-criteria:
1363                             -   name: "cpu_util_above_threshold"
1364                                 scale-in-threshold: 15
1365                                 scale-in-relational-operation: "LE"
1366                                 scale-out-threshold: 60
1367                                 scale-out-relational-operation: "GE"
1368                                 vnf-monitoring-param-ref: "monitor1"
1369                         vdu:
1370                         -   vdu-id-ref: dataVM
1371                             count: 1
1372                         scaling-config-action:
1373                         -   trigger: post-scale-out
1374                             vnf-config-primitive-name-ref: touch
1375                         -   trigger: pre-scale-in
1376                             vnf-config-primitive-name-ref: touch
1377                 vdu:
1378                     "$id: dataVM":
1379                         monitoring-param:
1380                         -   id: "dataVM_cpu_util"
1381                             nfvi-metric: "cpu_utilization"
1382
1383                 monitoring-param:
1384                 -   id: "monitor1"
1385                     name: "monitor1"
1386                     aggregation-type: AVERAGE
1387                     vdu-monitoring-param:
1388                       vdu-ref: "dataVM"
1389                       vdu-monitoring-param-ref: "dataVM_cpu_util"
1390                 vnf-configuration:
1391                     initial-config-primitive:
1392                         "$[1]":
1393                             parameter:
1394                                 "$[0]":
1395                                     value: "<touch-filename>"   # default-value: /home/ubuntu/first-touch
1396                     # config-primitive:
1397                     #     "$[0]":
1398                     #         parameter:
1399                     #             "$[0]":
1400                     #                 default-value: "<touch-filename2>"
1401                 """)
1402         }
1403         self.ns_params = {
1404             "additionalParamsForVnf": [
1405                 {"member-vnf-index": "1", "additionalParams": {"touch-filename": "/home/ubuntu/first-touch-1",
1406                                                                "touch-filename2": "/home/ubuntu/second-touch-2"}},
1407                 {"member-vnf-index": "2", "additionalParams": {"touch-filename": "/home/ubuntu/first-touch-1",
1408                                                                "touch-filename2": "/home/ubuntu/second-touch-2"}},
1409             ]
1410         }
1411
1412     def additional_operations(self, engine, test_osm, manual_check):
1413         super().additional_operations(engine, test_osm, manual_check)
1414         if not test_osm:
1415             return
1416
1417         # 2 perform scale out
1418         payload = '{scaleType: SCALE_VNF, scaleVnfData: {scaleVnfType: SCALE_OUT, scaleByStepData: ' \
1419                   '{scaling-group-descriptor: scale_dataVM, member-vnf-index: "1"}}}'
1420         engine.test("Execute scale action over NS", "POST",
1421                     "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload,
1422                     201, r_headers_yaml_location_nslcmop, "yaml")
1423         nslcmop2_scale_out = engine.last_id
1424         engine.wait_operation_ready("ns", nslcmop2_scale_out, timeout_deploy)
1425         if manual_check:
1426             input('NS scale out done. Check that file /home/ubuntu/touched is present and new VM is created')
1427         if test_osm:
1428             commands = {'1': ['ls -lrt /home/ubuntu/touched', ]}
1429             self.test_ns(engine, test_osm, commands=commands)
1430             # TODO check automatic connection to scaled VM
1431
1432         # 2 perform scale in
1433         payload = '{scaleType: SCALE_VNF, scaleVnfData: {scaleVnfType: SCALE_IN, scaleByStepData: ' \
1434                   '{scaling-group-descriptor: scale_dataVM, member-vnf-index: "1"}}}'
1435         engine.test("Execute scale action over NS", "POST",
1436                     "/nslcm/v1/ns_instances/{}/scale".format(self.ns_id), headers_yaml, payload,
1437                     201, r_headers_yaml_location_nslcmop, "yaml")
1438         nslcmop2_scale_in = engine.last_id
1439         engine.wait_operation_ready("ns", nslcmop2_scale_in, timeout_deploy)
1440         if manual_check:
1441             input('NS scale in done. Check that file /home/ubuntu/touched is updated and new VM is deleted')
1442         # TODO check automatic
1443
1444
1445 class TestDeploySimpleCharm(TestDeploy):
1446     description = "Deploy hackfest-4 hackfest_simplecharm example"
1447
1448     def __init__(self):
1449         super().__init__()
1450         self.test_name = "HACKFEST-SIMPLE"
1451         self.descriptor_url = "https://osm-download.etsi.org/ftp/osm-4.0-four/4th-hackfest/packages/"
1452         self.vnfd_filenames = ("hackfest_simplecharm_vnf.tar.gz",)
1453         self.nsd_filename = "hackfest_simplecharm_ns.tar.gz"
1454         self.uses_configuration = True
1455         self.commands = {'1': [''], '2': ['ls -lrt /home/ubuntu/first-touch', ]}
1456         self.users = {'1': "ubuntu", '2': "ubuntu"}
1457         self.passwords = {'1': "osm4u", '2': "osm4u"}
1458
1459
1460 class TestDeploySimpleCharm2(TestDeploySimpleCharm):
1461     description = "Deploy hackfest-4 hackfest_simplecharm example changing naming to contain dots on ids and " \
1462                   "vnf-member-index"
1463
1464     def __init__(self):
1465         super().__init__()
1466         self.test_name = "HACKFEST-SIMPLE2-"
1467         self.qforce = "?FORCE=True"
1468         self.descriptor_edit = {
1469             "vnfd0": {
1470                 "id": "hackfest.simplecharm.vnf"
1471             },
1472
1473             "nsd": {
1474                 "id": "hackfest.simplecharm.ns",
1475                 "constituent-vnfd": {
1476                     "$[0]": {"vnfd-id-ref": "hackfest.simplecharm.vnf", "member-vnf-index": "$1"},
1477                     "$[1]": {"vnfd-id-ref": "hackfest.simplecharm.vnf", "member-vnf-index": "$2"},
1478                 },
1479                 "vld": {
1480                     "$[0]": {
1481                         "vnfd-connection-point-ref": {"$[0]": {"member-vnf-index-ref": "$1",
1482                                                                "vnfd-id-ref": "hackfest.simplecharm.vnf"},
1483                                                       "$[1]": {"member-vnf-index-ref": "$2",
1484                                                                "vnfd-id-ref": "hackfest.simplecharm.vnf"}},
1485                     },
1486                     "$[1]": {
1487                         "vnfd-connection-point-ref": {"$[0]": {"member-vnf-index-ref": "$1",
1488                                                                "vnfd-id-ref": "hackfest.simplecharm.vnf"},
1489                                                       "$[1]": {"member-vnf-index-ref": "$2",
1490                                                                "vnfd-id-ref": "hackfest.simplecharm.vnf"}},
1491                     },
1492                 }
1493             }
1494         }
1495
1496
1497 class TestDeploySingleVdu(TestDeployHackfest3Charmed):
1498     description = "Generate a single VDU base on editing Hackfest3Charmed descriptors and deploy"
1499
1500     def __init__(self):
1501         super().__init__()
1502         self.test_name = "SingleVDU"
1503         self.qforce = "?FORCE=True"
1504         self.descriptor_edit = {
1505             # Modify VNFD to remove one VDU
1506             "vnfd0": {
1507                 "vdu": {
1508                     "$[0]": {
1509                         "interface": {"$[0]": {"external-connection-point-ref": "pdu-mgmt"}}
1510                     },
1511                     "$[1]": None
1512                 },
1513                 "vnf-configuration": None,
1514                 "connection-point": {
1515                     "$[0]": {
1516                         "id": "pdu-mgmt",
1517                         "name": "pdu-mgmt",
1518                         "short-name": "pdu-mgmt"
1519                     },
1520                     "$[1]": None
1521                 },
1522                 "mgmt-interface": {"cp": "pdu-mgmt"},
1523                 "description": "A vnf single vdu to be used as PDU",
1524                 "id": "vdu-as-pdu",
1525                 "internal-vld": {
1526                     "$[0]": {
1527                         "id": "pdu_internal",
1528                         "name": "pdu_internal",
1529                         "internal-connection-point": {"$[1]": None},
1530                         "short-name": "pdu_internal",
1531                         "type": "ELAN"
1532                     }
1533                 }
1534             },
1535
1536             # Modify NSD accordingly
1537             "nsd": {
1538                 "constituent-vnfd": {
1539                     "$[0]": {"vnfd-id-ref": "vdu-as-pdu"},
1540                     "$[1]": None,
1541                 },
1542                 "description": "A nsd to deploy the vnf to act as as PDU",
1543                 "id": "nsd-as-pdu",
1544                 "name": "nsd-as-pdu",
1545                 "short-name": "nsd-as-pdu",
1546                 "vld": {
1547                     "$[0]": {
1548                         "id": "mgmt_pdu",
1549                         "name": "mgmt_pdu",
1550                         "short-name": "mgmt_pdu",
1551                         "vnfd-connection-point-ref": {
1552                             "$[0]": {
1553                                 "vnfd-connection-point-ref": "pdu-mgmt",
1554                                 "vnfd-id-ref": "vdu-as-pdu",
1555                             },
1556                             "$[1]": None
1557                         },
1558                         "type": "ELAN"
1559                     },
1560                     "$[1]": None,
1561                 }
1562             }
1563         }
1564
1565
1566 class TestDeployHnfd(TestDeployHackfest3Charmed):
1567     description = "Generate a HNFD base on editing Hackfest3Charmed descriptors and deploy"
1568
1569     def __init__(self):
1570         super().__init__()
1571         self.test_name = "HNFD"
1572         self.pduDeploy = TestDeploySingleVdu()
1573         self.pdu_interface_0 = {}
1574         self.pdu_interface_1 = {}
1575
1576         self.pdu_id = None
1577         # self.vnf_to_pdu = """
1578         #     vdu:
1579         #         "$[0]":
1580         #             pdu-type: PDU-TYPE-1
1581         #             interface:
1582         #                 "$[0]":
1583         #                     name: mgmt-iface
1584         #                 "$[1]":
1585         #                     name: pdu-iface-internal
1586         #     id: hfn1
1587         #     description: HFND, one PDU + One VDU
1588         #     name: hfn1
1589         #     short-name: hfn1
1590         #
1591         # """
1592
1593         self.pdu_descriptor = {
1594             "name": "my-PDU",
1595             "type": "PDU-TYPE-1",
1596             "vim_accounts": "to-override",
1597             "interfaces": [
1598                 {
1599                     "name": "mgmt-iface",
1600                     "mgmt": True,
1601                     "type": "overlay",
1602                     "ip-address": "to override",
1603                     "mac-address": "mac_address",
1604                     "vim-network-name": "mgmt",
1605                 },
1606                 {
1607                     "name": "pdu-iface-internal",
1608                     "mgmt": False,
1609                     "type": "overlay",
1610                     "ip-address": "to override",
1611                     "mac-address": "mac_address",
1612                     "vim-network-name": "pdu_internal",  # OSMNBITEST-PDU-pdu_internal
1613                 },
1614             ]
1615         }
1616         self.vnfd_filenames = ("hackfest_3charmed_vnfd.tar.gz", "hackfest_3charmed_vnfd.tar.gz")
1617
1618         self.descriptor_edit = {
1619             "vnfd0": {
1620                 "id": "hfnd1",
1621                 "name": "hfn1",
1622                 "short-name": "hfn1",
1623                 "vdu": {
1624                     "$[0]": {
1625                         "pdu-type": "PDU-TYPE-1",
1626                         "interface": {
1627                             "$[0]": {"name": "mgmt-iface"},
1628                             "$[1]": {"name": "pdu-iface-internal"},
1629                         }
1630                     }
1631                 }
1632             },
1633             "nsd": {
1634                 "constituent-vnfd": {
1635                     "$[1]": {"vnfd-id-ref": "hfnd1"}
1636                 },
1637                 "vld": {
1638                     "$[0]": {"vnfd-connection-point-ref": {"$[1]": {"vnfd-id-ref": "hfnd1"}}},
1639                     "$[1]": {"vnfd-connection-point-ref": {"$[1]": {"vnfd-id-ref": "hfnd1"}}}
1640                 }
1641             }
1642         }
1643
1644     def create_descriptors(self, engine):
1645         super().create_descriptors(engine)
1646
1647         # Create PDU
1648         self.pdu_descriptor["interfaces"][0].update(self.pdu_interface_0)
1649         self.pdu_descriptor["interfaces"][1].update(self.pdu_interface_1)
1650         self.pdu_descriptor["vim_accounts"] = [self.vim_id]
1651         # TODO get vim-network-name from vnfr.vld.name
1652         self.pdu_descriptor["interfaces"][1]["vim-network-name"] = "{}-{}-{}".format(
1653             os.environ.get("OSMNBITEST_NS_NAME", "OSMNBITEST"),
1654             "PDU", self.pdu_descriptor["interfaces"][1]["vim-network-name"])
1655         engine.test("Onboard PDU descriptor", "POST", "/pdu/v1/pdu_descriptors",
1656                     {"Location": "/pdu/v1/pdu_descriptors/", "Content-Type": "application/yaml"}, self.pdu_descriptor,
1657                     201, r_header_yaml, "yaml")
1658         self.pdu_id = engine.last_id
1659
1660     def run(self, engine, test_osm, manual_check, test_params=None):
1661         engine.get_autorization()
1662         engine.set_test_name(self.test_name)
1663         nsname = os.environ.get("OSMNBITEST_NS_NAME", "OSMNBITEST")
1664
1665         # create real VIM if not exist
1666         self.vim_id = engine.get_create_vim(test_osm)
1667         # instantiate PDU
1668         self.pduDeploy.create_descriptors(engine)
1669         self.pduDeploy.instantiate(engine, {"nsDescription": "to be used as PDU", "nsName": nsname + "-PDU",
1670                                             "nsdId": self.pduDeploy.nsd_id, "vimAccountId": self.vim_id})
1671         if manual_check:
1672             input('VNF to be used as PDU has been deployed. Perform manual check and press enter to resume')
1673         if test_osm:
1674             self.pduDeploy.test_ns(engine, test_osm)
1675
1676         if test_osm:
1677             r = engine.test("Get VNFR to obtain IP_ADDRESS", "GET",
1678                             "/nslcm/v1/vnfrs?nsr-id-ref={}".format(self.pduDeploy.ns_id), headers_json, None,
1679                             200, r_header_json, "json")
1680             if not r:
1681                 return
1682             vnfr_data = r.json()
1683             # print(vnfr_data)
1684
1685             self.pdu_interface_0["ip-address"] = vnfr_data[0]["vdur"][0]["interfaces"][0].get("ip-address")
1686             self.pdu_interface_1["ip-address"] = vnfr_data[0]["vdur"][0]["interfaces"][1].get("ip-address")
1687             self.pdu_interface_0["mac-address"] = vnfr_data[0]["vdur"][0]["interfaces"][0].get("mac-address")
1688             self.pdu_interface_1["mac-address"] = vnfr_data[0]["vdur"][0]["interfaces"][1].get("mac-address")
1689             if not self.pdu_interface_0["ip-address"]:
1690                 raise TestException("Vnfr has not managment ip address")
1691         else:
1692             self.pdu_interface_0["ip-address"] = "192.168.10.10"
1693             self.pdu_interface_1["ip-address"] = "192.168.11.10"
1694             self.pdu_interface_0["mac-address"] = "52:33:44:55:66:13"
1695             self.pdu_interface_1["mac-address"] = "52:33:44:55:66:14"
1696
1697         self.create_descriptors(engine)
1698
1699         ns_data = {"nsDescription": "default description", "nsName": nsname, "nsdId": self.nsd_id,
1700                    "vimAccountId": self.vim_id}
1701         if test_params and test_params.get("ns-config"):
1702             if isinstance(test_params["ns-config"], str):
1703                 ns_data.update(yaml.load(test_params["ns-config"]))
1704             else:
1705                 ns_data.update(test_params["ns-config"])
1706
1707         self.instantiate(engine, ns_data)
1708         if manual_check:
1709             input('NS has been deployed. Perform manual check and press enter to resume')
1710         if test_osm:
1711             self.test_ns(engine, test_osm)
1712         self.additional_operations(engine, test_osm, manual_check)
1713         self.terminate(engine)
1714         self.pduDeploy.terminate(engine)
1715         self.delete_descriptors(engine)
1716         self.pduDeploy.delete_descriptors(engine)
1717
1718     def delete_descriptors(self, engine):
1719         super().delete_descriptors(engine)
1720         # delete pdu
1721         engine.test("Delete PDU SOL005", "DELETE",
1722                     "/pdu/v1/pdu_descriptors/{}".format(self.pdu_id),
1723                     headers_yaml, None, 204, None, 0)
1724
1725
1726 class TestDescriptors:
1727     description = "Test VNFD, NSD, PDU descriptors CRUD and dependencies"
1728
1729     def __init__(self):
1730         self.vnfd_filename = "hackfest_3charmed_vnfd.tar.gz"
1731         self.nsd_filename = "hackfest_3charmed_nsd.tar.gz"
1732         self.descriptor_url = "https://osm-download.etsi.org/ftp/osm-3.0-three/2nd-hackfest/packages/"
1733         self.vnfd_id = None
1734         self.nsd_id = None
1735         self.vnfd_empty = """vnfd:vnfd-catalog:
1736             vnfd:
1737             -   name: prova
1738                 short-name: prova
1739                 id: prova
1740         """
1741         self.vnfd_prova = """vnfd:vnfd-catalog:
1742             vnfd:
1743             -   connection-point:
1744                 -   name: cp_0h8m
1745                     type: VPORT
1746                 id: prova
1747                 name: prova
1748                 short-name: prova
1749                 vdu:
1750                 -   id: vdu_z4bm
1751                     image: ubuntu
1752                     interface:
1753                     -   external-connection-point-ref: cp_0h8m
1754                         name: eth0
1755                         virtual-interface:
1756                         type: VIRTIO
1757                     name: vdu_z4bm
1758                 version: '1.0'
1759         """
1760
1761     def run(self, engine, test_osm, manual_check, test_params=None):
1762         engine.set_test_name("Descriptors")
1763         engine.get_autorization()
1764         temp_dir = os.path.dirname(os.path.abspath(__file__)) + "/temp/"
1765         if not os.path.exists(temp_dir):
1766             os.makedirs(temp_dir)
1767
1768         # download files
1769         for filename in (self.vnfd_filename, self.nsd_filename):
1770             filename_path = temp_dir + filename
1771             if not os.path.exists(filename_path):
1772                 with open(filename_path, "wb") as file:
1773                     response = requests.get(self.descriptor_url + filename)
1774                     if response.status_code >= 300:
1775                         raise TestException("Error downloading descriptor from '{}': {}".format(
1776                             self.descriptor_url + filename, response.status_code))
1777                     file.write(response.content)
1778
1779         vnfd_filename_path = temp_dir + self.vnfd_filename
1780         nsd_filename_path = temp_dir + self.nsd_filename
1781
1782         engine.test("Onboard empty VNFD in one step", "POST", "/vnfpkgm/v1/vnf_packages_content", headers_yaml,
1783                     self.vnfd_empty, 201, r_headers_yaml_location_vnfd, "yaml")
1784         self.vnfd_id = engine.last_id
1785
1786         # test bug 605
1787         engine.test("Upload invalid VNFD ", "PUT", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(self.vnfd_id),
1788                     headers_yaml, self.vnfd_prova, 422, r_header_yaml, "yaml")
1789
1790         engine.test("Upload VNFD {}".format(self.vnfd_filename), "PUT",
1791                     "/vnfpkgm/v1/vnf_packages/{}/package_content".format(self.vnfd_id), headers_zip_yaml,
1792                     "@b" + vnfd_filename_path, 204, None, 0)
1793
1794         # test bug 605
1795         engine.test("Upload invalid VNFD ", "PUT", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(self.vnfd_id),
1796                     headers_yaml, self.vnfd_prova, 422, r_header_yaml, "yaml")
1797
1798         # get vnfd descriptor
1799         engine.test("Get VNFD descriptor", "GET", "/vnfpkgm/v1/vnf_packages/{}".format(self.vnfd_id),
1800                     headers_yaml, None, 200, r_header_yaml, "yaml")
1801
1802         # get vnfd file descriptor
1803         engine.test("Get VNFD file descriptor", "GET", "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(self.vnfd_id),
1804                     headers_text, None, 200, r_header_text, "text", temp_dir+"vnfd-yaml")
1805         # TODO compare files: diff vnfd-yaml hackfest_3charmed_vnfd/hackfest_3charmed_vnfd.yaml
1806
1807         # get vnfd zip file package
1808         engine.test("Get VNFD zip package", "GET",
1809                     "/vnfpkgm/v1/vnf_packages/{}/package_content".format(self.vnfd_id), headers_zip, None, 200,
1810                     r_header_zip, "zip", temp_dir+"vnfd-zip")
1811         # TODO compare files: diff vnfd-zip hackfest_3charmed_vnfd.tar.gz
1812
1813         # get vnfd artifact
1814         engine.test("Get VNFD artifact package", "GET",
1815                     "/vnfpkgm/v1/vnf_packages/{}/artifacts/icons/osm.png".format(self.vnfd_id), headers_zip, None, 200,
1816                     r_header_octect, "octet-string", temp_dir+"vnfd-icon")
1817         # TODO compare files: diff vnfd-icon hackfest_3charmed_vnfd/icons/osm.png
1818
1819         # nsd CREATE AND UPLOAD in one step:
1820         engine.test("Onboard NSD in one step", "POST", "/nsd/v1/ns_descriptors_content", headers_zip_yaml,
1821                     "@b" + nsd_filename_path, 201, r_headers_yaml_location_nsd, "yaml")
1822         self.nsd_id = engine.last_id
1823
1824         # get nsd descriptor
1825         engine.test("Get NSD descriptor", "GET", "/nsd/v1/ns_descriptors/{}".format(self.nsd_id), headers_yaml,
1826                     None, 200, r_header_yaml, "yaml")
1827
1828         # get nsd file descriptor
1829         engine.test("Get NSD file descriptor", "GET", "/nsd/v1/ns_descriptors/{}/nsd".format(self.nsd_id), headers_text,
1830                     None, 200, r_header_text, "text", temp_dir+"nsd-yaml")
1831         # TODO compare files: diff nsd-yaml hackfest_3charmed_nsd/hackfest_3charmed_nsd.yaml
1832
1833         # get nsd zip file package
1834         engine.test("Get NSD zip package", "GET", "/nsd/v1/ns_descriptors/{}/nsd_content".format(self.nsd_id),
1835                     headers_zip, None, 200, r_header_zip, "zip", temp_dir+"nsd-zip")
1836         # TODO compare files: diff nsd-zip hackfest_3charmed_nsd.tar.gz
1837
1838         # get nsd artifact
1839         engine.test("Get NSD artifact package", "GET",
1840                     "/nsd/v1/ns_descriptors/{}/artifacts/icons/osm.png".format(self.nsd_id), headers_zip, None, 200,
1841                     r_header_octect, "octet-string", temp_dir+"nsd-icon")
1842         # TODO compare files: diff nsd-icon hackfest_3charmed_nsd/icons/osm.png
1843
1844         # vnfd DELETE
1845         test_rest.test("Delete VNFD conflict", "DELETE", "/vnfpkgm/v1/vnf_packages/{}".format(self.vnfd_id),
1846                        headers_yaml, None, 409, None, None)
1847
1848         test_rest.test("Delete VNFD force", "DELETE", "/vnfpkgm/v1/vnf_packages/{}?FORCE=TRUE".format(self.vnfd_id),
1849                        headers_yaml, None, 204, None, 0)
1850
1851         # nsd DELETE
1852         test_rest.test("Delete NSD", "DELETE", "/nsd/v1/ns_descriptors/{}".format(self.nsd_id), headers_yaml, None, 204,
1853                        None, 0)
1854
1855
1856 class TestNetSliceTemplates:
1857     description = "Upload a NST to OSM"
1858
1859     def __init__(self):
1860         self.nst_filenames = ("@./cirros_slice/cirros_slice_vld.yaml")
1861
1862     def run(self, engine, test_osm, manual_check, test_params=None):
1863         # nst CREATE
1864         engine.set_test_name("NST")
1865         engine.get_autorization()
1866         engine.test("Onboard NST", "POST", "/nst/v1/netslice_templates_content", headers_yaml, self.nst_filenames,
1867                     201, r_headers_yaml_location_nst, "yaml")
1868         nst_id = engine.last_id
1869
1870         # nstd SHOW OSM format
1871         engine.test("Show NSTD OSM format", "GET", "/nst/v1/netslice_templates/{}".format(nst_id), headers_json, None,
1872                     200, r_header_json, "json")
1873
1874         # nstd DELETE
1875         engine.test("Delete NSTD", "DELETE", "/nst/v1/netslice_templates/{}".format(nst_id), headers_json, None,
1876                     204, None, 0)
1877
1878
1879 class TestNetSliceInstances:
1880     description = "Upload a NST to OSM"
1881
1882     def __init__(self):
1883         self.vim_id = None
1884         self.nst_filenames = ("@./cirros_slice/cirros_slice.yaml")
1885
1886     def run(self, engine, test_osm, manual_check, test_params=None):
1887         # nst CREATE
1888         engine.set_test_name("NSI")
1889         engine.get_autorization()
1890         engine.test("Onboard NST", "POST", "/nst/v1/netslice_templates_content", headers_yaml, self.nst_filenames, 201,
1891                     r_headers_yaml_location_nst, "yaml")
1892         nst_id = engine.last_id
1893
1894         # nsi CREATE
1895         self.vim_id = engine.get_create_vim(test_osm)
1896
1897         ns_data = {"nsiDescription": "default description", "nsiName": "my_slice", "nstId": nst_id,
1898                    "vimAccountId": self.vim_id}
1899         ns_data_text = yaml.safe_dump(ns_data, default_flow_style=True, width=256)
1900
1901         engine.test("Onboard NSI", "POST", "/nsilcm/v1/netslice_instances_content", headers_yaml, ns_data_text, 201,
1902                     r_headers_yaml_location_nst, "yaml")
1903         nsi_id = engine.last_id
1904
1905         # TODO: Improve the wait with a polling if NSI was deployed
1906         wait = 120
1907         sleep(wait)
1908
1909         # Check deployment
1910         engine.test("Wait until NSI is deployed", "GET", "/nsilcm/v1/netslice_instances_content/{}".format(nsi_id),
1911                     headers_json, None, 200, r_header_json, "json")
1912
1913         # nsi DELETE
1914         engine.test("Delete NSI", "DELETE", "/nsilcm/v1/netslice_instances_content/{}".format(nsi_id), headers_json,
1915                     None, 202, r_header_json, "json")
1916
1917         sleep(60)
1918
1919         # nstd DELETE
1920         engine.test("Delete NSTD", "DELETE", "/nst/v1/netslice_templates/{}".format(nst_id), headers_json, None,
1921                     204, None, 0)
1922
1923
1924 if __name__ == "__main__":
1925     global logger
1926     test = ""
1927
1928     # Disable warnings from self-signed certificates.
1929     requests.packages.urllib3.disable_warnings()
1930     try:
1931         logging.basicConfig(format="%(levelname)s %(message)s", level=logging.ERROR)
1932         logger = logging.getLogger('NBI')
1933         # load parameters and configuration
1934         opts, args = getopt.getopt(sys.argv[1:], "hvu:p:",
1935                                    ["url=", "user=", "password=", "help", "version", "verbose", "no-verbose",
1936                                     "project=", "insecure", "timeout", "timeout-deploy", "timeout-configure",
1937                                     "test=", "list", "test-osm", "manual-check", "params=", 'fail-fast'])
1938         url = "https://localhost:9999/osm"
1939         user = password = project = "admin"
1940         test_osm = False
1941         manual_check = False
1942         verbose = 0
1943         verify = True
1944         fail_fast = False
1945         test_classes = {
1946             "NonAuthorized": TestNonAuthorized,
1947             "FakeVIM": TestFakeVim,
1948             "TestUsersProjects": TestUsersProjects,
1949             "VIM-SDN": TestVIMSDN,
1950             "Deploy-Custom": TestDeploy,
1951             "Deploy-Hackfest-Cirros": TestDeployHackfestCirros,
1952             "Deploy-Hackfest-Cirros-Scaling": TestDeployHackfestCirrosScaling,
1953             "Deploy-Hackfest-3Charmed": TestDeployHackfest3Charmed,
1954             "Deploy-Hackfest-3Charmed2": TestDeployHackfest3Charmed2,
1955             "Deploy-Hackfest-3Charmed3": TestDeployHackfest3Charmed3,
1956             "Deploy-Hackfest-4": TestDeployHackfest4,
1957             "Deploy-CirrosMacIp": TestDeployIpMac,
1958             "TestDescriptors": TestDescriptors,
1959             "TestDeployHackfest1": TestDeployHackfest1,
1960             # "Deploy-MultiVIM": TestDeployMultiVIM,
1961             "DeploySingleVdu": TestDeploySingleVdu,
1962             "DeployHnfd": TestDeployHnfd,
1963             # "Upload-Slice-Template": TestNetSliceTemplates,
1964             # "Deploy-Slice-Instance": TestNetSliceInstances,
1965             "TestDeploySimpleCharm": TestDeploySimpleCharm,
1966             "TestDeploySimpleCharm2": TestDeploySimpleCharm2,
1967         }
1968         test_to_do = []
1969         test_params = {}
1970
1971         for o, a in opts:
1972             # print("parameter:", o, a)
1973             if o == "--version":
1974                 print("test version " + __version__ + ' ' + version_date)
1975                 exit()
1976             elif o == "--list":
1977                 for test, test_class in sorted(test_classes.items()):
1978                     print("{:32} {}".format(test + ":", test_class.description))
1979                 exit()
1980             elif o in ("-v", "--verbose"):
1981                 verbose += 1
1982             elif o == "no-verbose":
1983                 verbose = -1
1984             elif o in ("-h", "--help"):
1985                 usage()
1986                 sys.exit()
1987             elif o == "--test-osm":
1988                 test_osm = True
1989             elif o == "--manual-check":
1990                 manual_check = True
1991             elif o == "--url":
1992                 url = a
1993             elif o in ("-u", "--user"):
1994                 user = a
1995             elif o in ("-p", "--password"):
1996                 password = a
1997             elif o == "--project":
1998                 project = a
1999             elif o == "--fail-fast":
2000                 fail_fast = True
2001             elif o == "--test":
2002                 # print("asdfadf", o, a, a.split(","))
2003                 for _test in a.split(","):
2004                     if _test not in test_classes:
2005                         print("Invalid test name '{}'. Use option '--list' to show available tests".format(_test),
2006                               file=sys.stderr)
2007                         exit(1)
2008                     test_to_do.append(_test)
2009             elif o == "--params":
2010                 param_key, _, param_value = a.partition("=")
2011                 text_index = len(test_to_do)
2012                 if text_index not in test_params:
2013                     test_params[text_index] = {}
2014                 test_params[text_index][param_key] = param_value
2015             elif o == "--insecure":
2016                 verify = False
2017             elif o == "--timeout":
2018                 timeout = int(a)
2019             elif o == "--timeout-deploy":
2020                 timeout_deploy = int(a)
2021             elif o == "--timeout-configure":
2022                 timeout_configure = int(a)
2023             else:
2024                 assert False, "Unhandled option"
2025         if verbose == 0:
2026             logger.setLevel(logging.WARNING)
2027         elif verbose > 1:
2028             logger.setLevel(logging.DEBUG)
2029         else:
2030             logger.setLevel(logging.ERROR)
2031
2032         test_rest = TestRest(url, user=user, password=password, project=project)
2033         # print("tests to do:", test_to_do)
2034         if test_to_do:
2035             text_index = 0
2036             for test in test_to_do:
2037                 if fail_fast and test_rest.failed_tests:
2038                     break
2039                 text_index += 1
2040                 test_class = test_classes[test]
2041                 test_class().run(test_rest, test_osm, manual_check, test_params.get(text_index))
2042         else:
2043             for test, test_class in test_classes.items():
2044                 if fail_fast and test_rest.failed_tests:
2045                     break
2046                 test_class().run(test_rest, test_osm, manual_check, test_params.get(0))
2047         test_rest.print_results()
2048         exit(1 if test_rest.failed_tests else 0)
2049
2050     except TestException as e:
2051         logger.error(test + "Test {} Exception: {}".format(test, str(e)))
2052         exit(1)
2053     except getopt.GetoptError as e:
2054         logger.error(e)
2055         print(e, file=sys.stderr)
2056         exit(1)
2057     except Exception as e:
2058         logger.critical(test + " Exception: " + str(e), exc_info=True)