RO/osm_ro/nfvo.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 NFVO engine, implementing all the methods for the creation, deletion and management of vnfs, scenarios and instances
  26 '''
  27 __author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes"
  28 __date__ ="$16-sep-2014 22:05:01$"
  29
  30 # import imp
  31 import json
  32 import yaml
  33 from random import choice as random_choice
  34 from osm_ro import utils
  35 from osm_ro.utils import deprecated
  36 from osm_ro.vim_thread import vim_thread
  37 import osm_ro.console_proxy_thread as cli
  38 from osm_ro import vimconn
  39 import logging
  40 import collections
  41 import math
  42 from uuid import uuid4
  43 from osm_ro.db_base import db_base_Exception
  44
  45 from osm_ro import nfvo_db
  46 from threading import Lock
  47 import time as t
  48 # TODO py3 BEGIN
  49 from osm_ro.sdn import Sdn, SdnException as ovimException
  50 # from lib_osm_openvim.ovim import ovimException
  51 # from unittest.mock  import MagicMock
  52 # class ovimException(Exception):
  53 #    pass
  54 # TODO py3 END
  55
  56 from Crypto.PublicKey import RSA
  57
  58 import osm_im.vnfd as vnfd_catalog
  59 import osm_im.nsd as nsd_catalog
  60 from pyangbind.lib.serialise import pybindJSONDecoder
  61 from copy import deepcopy
  62 from pkg_resources import iter_entry_points
  63
  64
  65 # WIM
  66 from .wim import sdnconn
  67 from .wim.wimconn_dummy import DummyConnector
  68 from .wim.failing_connector import FailingConnector
  69 from .http_tools import errors as httperrors
  70 from .wim.engine import WimEngine
  71 from .wim.persistence import WimPersistence
  72 from copy import deepcopy
  73 from pprint import pformat
  74 #
  75
  76 global global_config
  77 # WIM
  78 global wim_engine
  79 wim_engine  = None
  80 global sdnconn_imported
  81 #
  82 global logger
  83 global default_volume_size
  84 default_volume_size = '5' #size in GB
  85 global ovim
  86 ovim = None
  87 global_config = None
  88
  89 plugins = {}   # dictionary with VIM type as key, loaded module as value
  90 vim_threads = {"running":{}, "deleting": {}, "names": []}      # threads running for attached-VIMs
  91 vim_persistent_info = {}
  92 # WIM
  93 sdnconn_imported = {}   # dictionary with WIM type as key, loaded module as value
  94 wim_threads = {"running":{}, "deleting": {}, "names": []}      # threads running for attached-WIMs
  95 wim_persistent_info = {}
  96 #
  97
  98 logger = logging.getLogger('openmano.nfvo')
  99 task_lock = Lock()
 100 last_task_id = 0.0
 101 db = None
 102 db_lock = Lock()
 103
 104 worker_id = None
 105
 106 class NfvoException(httperrors.HttpMappedError):
 107     """Common Class for NFVO errors"""
 108
 109 def _load_plugin(name, type="vim"):
 110     # type can be vim or sdn
 111     global plugins
 112     try:
 113         for v in iter_entry_points('osm_ro{}.plugins'.format(type), name):
 114             plugins[name] = v.load()
 115     except Exception as e:
 116         logger.critical("Cannot load osm_{}: {}".format(name, e))
 117         if name:
 118             plugins[name] = FailingConnector("Cannot load osm_{}: {}".format(name, e))
 119     if name and name not in plugins:
 120         error_text = "Cannot load a module for {t} type '{n}'. The plugin 'osm_{n}' has not been" \
 121                      " registered".format(t=type, n=name)
 122         logger.critical(error_text)
 123         plugins[name] = FailingConnector(error_text)
 124         # raise NfvoException("Cannot load a module for {t} type '{n}'. The plugin 'osm_{n}' has not been registered".
 125         #                     format(t=type, n=name), httperrors.Bad_Request)
 126
 127 def get_task_id():
 128     global last_task_id
 129     task_id = t.time()
 130     if task_id <= last_task_id:
 131         task_id = last_task_id + 0.000001
 132     last_task_id = task_id
 133     return "ACTION-{:.6f}".format(task_id)
 134     # return (t.strftime("%Y%m%dT%H%M%S.{}%Z", t.localtime(task_id))).format(int((task_id % 1)*1e6))
 135
 136
 137 def new_task(name, params, depends=None):
 138     """Deprected!!!"""
 139     task_id = get_task_id()
 140     task = {"status": "enqueued", "id": task_id, "name": name, "params": params}
 141     if depends:
 142         task["depends"] = depends
 143     return task
 144
 145
 146 def is_task_id(id):
 147     return True if id[:5] == "TASK-" else False
 148
 149 def get_process_id():
 150     """
 151     Obtain a unique ID for this process. If running from inside docker, it will get docker ID. If not it
 152     will provide a random one
 153     :return: Obtained ID
 154     """
 155     # Try getting docker id. If fails, get pid
 156     try:
 157         with open("/proc/self/cgroup", "r") as f:
 158             text_id_ = f.readline()
 159             _, _, text_id = text_id_.rpartition("/")
 160             text_id = text_id.replace("\n", "")[:12]
 161             if text_id:
 162                 return text_id
 163     except Exception:
 164         pass
 165     # Return a random id
 166     return "".join(random_choice("0123456789abcdef") for _ in range(12))
 167
 168 def get_non_used_vim_name(datacenter_name, datacenter_id):
 169     return "{}:{}:{}".format(
 170         worker_id[:12], datacenter_id.replace("-", "")[:32], datacenter_name[:16]
 171     )
 172
 173 # -- Move
 174 def get_non_used_wim_name(wim_name, wim_id, tenant_name, tenant_id):
 175     name = wim_name[:16]
 176     if name not in wim_threads["names"]:
 177         wim_threads["names"].append(name)
 178         return name
 179     name = wim_name[:16] + "." + tenant_name[:16]
 180     if name not in wim_threads["names"]:
 181         wim_threads["names"].append(name)
 182         return name
 183     name = wim_id + "-" + tenant_id
 184     wim_threads["names"].append(name)
 185     return name
 186
 187
 188 def start_service(mydb, persistence=None, wim=None):
 189     global db, global_config, plugins, ovim, worker_id
 190     db = nfvo_db.nfvo_db(lock=db_lock)
 191     mydb.lock = db_lock
 192     db.connect(global_config['db_host'], global_config['db_user'], global_config['db_passwd'], global_config['db_name'])
 193
 194     persistence = persistence or  WimPersistence(db)
 195
 196     try:
 197         worker_id = get_process_id()
 198         if "rosdn_dummy" not in plugins:
 199             plugins["rosdn_dummy"] = DummyConnector
 200         # starts ovim library
 201         ovim = Sdn(db, plugins)
 202
 203         global wim_engine
 204         wim_engine = wim or WimEngine(persistence, plugins)
 205         wim_engine.ovim = ovim
 206
 207         ovim.start_service()
 208
 209         #delete old unneeded vim_wim_actions
 210         clean_db(mydb)
 211
 212         # starts vim_threads
 213         from_= 'tenants_datacenters as td join datacenters as d on td.datacenter_id=d.uuid join '\
 214                 'datacenter_tenants as dt on td.datacenter_tenant_id=dt.uuid'
 215         select_ = ('type', 'd.config as config', 'd.uuid as datacenter_id', 'vim_url', 'vim_url_admin',
 216                    'd.name as datacenter_name', 'dt.uuid as datacenter_tenant_id',
 217                    'dt.vim_tenant_name as vim_tenant_name', 'dt.vim_tenant_id as vim_tenant_id',
 218                    'user', 'passwd', 'dt.config as dt_config', 'nfvo_tenant_id')
 219         vims = mydb.get_rows(FROM=from_, SELECT=select_)
 220         for vim in vims:
 221             extra={'datacenter_tenant_id': vim.get('datacenter_tenant_id'),
 222                    'datacenter_id': vim.get('datacenter_id')}
 223             if vim["config"]:
 224                 extra.update(yaml.load(vim["config"], Loader=yaml.Loader))
 225             if vim.get('dt_config'):
 226                 extra.update(yaml.load(vim["dt_config"], Loader=yaml.Loader))
 227             plugin_name = "rovim_" + vim["type"]
 228             if plugin_name not in plugins:
 229                 _load_plugin(plugin_name, type="vim")
 230
 231             thread_id = vim['datacenter_tenant_id']
 232             vim_persistent_info[thread_id] = {}
 233             try:
 234                 #if not tenant:
 235                 #    return -httperrors.Bad_Request, "You must provide a valid tenant name or uuid for VIM  %s" % ( vim["type"])
 236                 myvim = plugins[plugin_name].vimconnector(
 237                     uuid=vim['datacenter_id'], name=vim['datacenter_name'],
 238                     tenant_id=vim['vim_tenant_id'], tenant_name=vim['vim_tenant_name'],
 239                     url=vim['vim_url'], url_admin=vim['vim_url_admin'],
 240                     user=vim['user'], passwd=vim['passwd'],
 241                     config=extra, persistent_info=vim_persistent_info[thread_id]
 242                 )
 243             except vimconn.vimconnException as e:
 244                 myvim = e
 245                 logger.error("Cannot launch thread for VIM {} '{}': {}".format(vim['datacenter_name'],
 246                                                                                vim['datacenter_id'], e))
 247             except Exception as e:
 248                 logger.critical("Cannot launch thread for VIM {} '{}': {}".format(vim['datacenter_name'],
 249                                                                                   vim['datacenter_id'], e))
 250                 # raise NfvoException("Error at VIM  {}; {}: {}".format(vim["type"], type(e).__name__, e),
 251                 #                     httperrors.Internal_Server_Error)
 252             thread_name = get_non_used_vim_name(vim['datacenter_name'], vim['datacenter_id'])
 253             new_thread = vim_thread(task_lock, plugins, thread_name, None,
 254                                     vim['datacenter_tenant_id'], db=db)
 255             new_thread.start()
 256             vim_threads["running"][thread_id] = new_thread
 257         wims = mydb.get_rows(FROM="wim_accounts join wims on wim_accounts.wim_id=wims.uuid",
 258                              WHERE={"sdn": "true"},
 259                              SELECT=("wim_accounts.uuid as uuid", "type", "wim_accounts.name as name"))
 260         for wim in wims:
 261             plugin_name = "rosdn_" + wim["type"]
 262             if plugin_name not in plugins:
 263                 _load_plugin(plugin_name, type="sdn")
 264
 265             thread_id = wim['uuid']
 266             thread_name = get_non_used_vim_name(wim['name'], wim['uuid'])
 267             new_thread = vim_thread(task_lock, plugins, thread_name, wim['uuid'], None, db=db)
 268             new_thread.start()
 269             vim_threads["running"][thread_id] = new_thread
 270         wim_engine.start_threads()
 271     except db_base_Exception as e:
 272         raise NfvoException(str(e) + " at nfvo.get_vim", e.http_code)
 273     except ovimException as e:
 274         message = str(e)
 275         if message[:22] == "DATABASE wrong version":
 276             message = "DATABASE wrong version of lib_osm_openvim {msg} -d{dbname} -u{dbuser} -p{dbpass} {ver}' "\
 277                       "at host {dbhost}".format(
 278                             msg=message[22:-3], dbname=global_config["db_ovim_name"],
 279                             dbuser=global_config["db_ovim_user"], dbpass=global_config["db_ovim_passwd"],
 280                             ver=message[-3:-1], dbhost=global_config["db_ovim_host"])
 281         raise NfvoException(message, httperrors.Bad_Request)
 282
 283
 284 def stop_service():
 285     global ovim, global_config
 286     if ovim:
 287         ovim.stop_service()
 288     for thread_id, thread in vim_threads["running"].items():
 289         thread.insert_task("exit")
 290         vim_threads["deleting"][thread_id] = thread
 291     vim_threads["running"] = {}
 292
 293     if wim_engine:
 294         wim_engine.stop_threads()
 295
 296     if global_config and global_config.get("console_thread"):
 297         for thread in global_config["console_thread"]:
 298             thread.terminate = True
 299
 300 def get_version():
 301     return  ("openmanod version {} {}\n(c) Copyright Telefonica".format(global_config["version"],
 302                                                                         global_config["version_date"] ))
 303
 304 def clean_db(mydb):
 305     """
 306     Clean unused or old entries at database to avoid unlimited growing
 307     :param mydb: database connector
 308     :return: None
 309     """
 310     # get and delete unused vim_wim_actions: all elements deleted, one week before, instance not present
 311     now = t.time()-3600*24*7
 312     instance_action_id = None
 313     nb_deleted = 0
 314     while True:
 315         actions_to_delete = mydb.get_rows(
 316             SELECT=("item", "item_id", "instance_action_id"),
 317             FROM="vim_wim_actions as va join instance_actions as ia on va.instance_action_id=ia.uuid "
 318                     "left join instance_scenarios as i on ia.instance_id=i.uuid",
 319             WHERE={"va.action": "DELETE", "va.modified_at<": now, "i.uuid": None,
 320                    "va.status": ("DONE", "SUPERSEDED")},
 321             LIMIT=100
 322         )
 323         for to_delete in actions_to_delete:
 324             mydb.delete_row(FROM="vim_wim_actions", WHERE=to_delete)
 325             if instance_action_id != to_delete["instance_action_id"]:
 326                 instance_action_id = to_delete["instance_action_id"]
 327                 mydb.delete_row(FROM="instance_actions", WHERE={"uuid": instance_action_id})
 328         nb_deleted += len(actions_to_delete)
 329         if len(actions_to_delete) < 100:
 330             break
 331     # clean locks
 332     mydb.update_rows("vim_wim_actions", UPDATE={"worker": None}, WHERE={"worker<>": None})
 333
 334     if nb_deleted:
 335         logger.debug("Removed {} unused vim_wim_actions".format(nb_deleted))
 336
 337
 338 def get_flavorlist(mydb, vnf_id, nfvo_tenant=None):
 339     '''Obtain flavorList
 340     return result, content:
 341         <0, error_text upon error
 342         nb_records, flavor_list on success
 343     '''
 344     WHERE_dict={}
 345     WHERE_dict['vnf_id'] = vnf_id
 346     if nfvo_tenant is not None:
 347         WHERE_dict['nfvo_tenant_id'] = nfvo_tenant
 348
 349     #result, content = mydb.get_table(FROM='vms join vnfs on vms.vnf_id = vnfs.uuid',SELECT=('uuid'),WHERE=WHERE_dict )
 350     #result, content = mydb.get_table(FROM='vms',SELECT=('vim_flavor_id',),WHERE=WHERE_dict )
 351     flavors = mydb.get_rows(FROM='vms join flavors on vms.flavor_id=flavors.uuid',SELECT=('flavor_id',),WHERE=WHERE_dict )
 352     #print "get_flavor_list result:", result
 353     #print "get_flavor_list content:", content
 354     flavorList=[]
 355     for flavor in flavors:
 356         flavorList.append(flavor['flavor_id'])
 357     return flavorList
 358
 359
 360 def get_imagelist(mydb, vnf_id, nfvo_tenant=None):
 361     """
 362     Get used images of all vms belonging to this VNFD
 363     :param mydb: database conector
 364     :param vnf_id: vnfd uuid
 365     :param nfvo_tenant: tenant, not used
 366     :return: The list of image uuid used
 367     """
 368     image_list = []
 369     vms = mydb.get_rows(SELECT=('image_id','image_list'), FROM='vms', WHERE={'vnf_id': vnf_id})
 370     for vm in vms:
 371         if vm["image_id"] and vm["image_id"] not in image_list:
 372             image_list.append(vm["image_id"])
 373         if vm["image_list"]:
 374             vm_image_list = yaml.load(vm["image_list"], Loader=yaml.Loader)
 375             for image_dict in vm_image_list:
 376                 if image_dict["image_id"] not in image_list:
 377                     image_list.append(image_dict["image_id"])
 378     return image_list
 379
 380
 381 def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, datacenter_tenant_id=None,
 382             vim_tenant=None, vim_tenant_name=None, vim_user=None, vim_passwd=None, ignore_errors=False):
 383     '''Obtain a dictionary of VIM (datacenter) classes with some of the input parameters
 384     return dictionary with {datacenter_id: vim_class, ... }. vim_class contain:
 385             'nfvo_tenant_id','datacenter_id','vim_tenant_id','vim_url','vim_url_admin','datacenter_name','type','user','passwd'
 386         raise exception upon error
 387     '''
 388     global plugins
 389     WHERE_dict={}
 390     if nfvo_tenant     is not None:  WHERE_dict['nfvo_tenant_id'] = nfvo_tenant
 391     if datacenter_id   is not None:  WHERE_dict['d.uuid']  = datacenter_id
 392     if datacenter_tenant_id is not None:  WHERE_dict['datacenter_tenant_id']  = datacenter_tenant_id
 393     if datacenter_name is not None:  WHERE_dict['d.name']  = datacenter_name
 394     if vim_tenant      is not None:  WHERE_dict['dt.vim_tenant_id']  = vim_tenant
 395     if vim_tenant_name is not None:  WHERE_dict['vim_tenant_name']  = vim_tenant_name
 396     if nfvo_tenant or vim_tenant or vim_tenant_name or datacenter_tenant_id:
 397         from_= 'tenants_datacenters as td join datacenters as d on td.datacenter_id=d.uuid join datacenter_tenants as dt on td.datacenter_tenant_id=dt.uuid'
 398         select_ = ('type','d.config as config','d.uuid as datacenter_id', 'vim_url', 'vim_url_admin', 'd.name as datacenter_name',
 399                    'dt.uuid as datacenter_tenant_id','dt.vim_tenant_name as vim_tenant_name','dt.vim_tenant_id as vim_tenant_id',
 400                    'user','passwd', 'dt.config as dt_config')
 401     else:
 402         from_ = 'datacenters as d'
 403         select_ = ('type','config','d.uuid as datacenter_id', 'vim_url', 'vim_url_admin', 'd.name as datacenter_name')
 404     try:
 405         vims = mydb.get_rows(FROM=from_, SELECT=select_, WHERE=WHERE_dict )
 406         vim_dict={}
 407         for vim in vims:
 408             extra={'datacenter_tenant_id': vim.get('datacenter_tenant_id'),
 409                    'datacenter_id': vim.get('datacenter_id'),
 410                    '_vim_type_internal': vim.get('type')}
 411             if vim["config"]:
 412                 extra.update(yaml.load(vim["config"], Loader=yaml.Loader))
 413             if vim.get('dt_config'):
 414                 extra.update(yaml.load(vim["dt_config"], Loader=yaml.Loader))
 415             plugin_name = "rovim_" + vim["type"]
 416             if plugin_name not in plugins:
 417                 try:
 418                     _load_plugin(plugin_name, type="vim")
 419                 except NfvoException as e:
 420                     if ignore_errors:
 421                         logger.error("{}".format(e))
 422                         continue
 423                     else:
 424                         raise
 425             try:
 426                 if 'datacenter_tenant_id' in vim:
 427                     thread_id = vim["datacenter_tenant_id"]
 428                     if thread_id not in vim_persistent_info:
 429                         vim_persistent_info[thread_id] = {}
 430                     persistent_info = vim_persistent_info[thread_id]
 431                 else:
 432                     persistent_info = {}
 433                 #if not tenant:
 434                 #    return -httperrors.Bad_Request, "You must provide a valid tenant name or uuid for VIM  %s" % ( vim["type"])
 435                 vim_dict[vim['datacenter_id']] = plugins[plugin_name].vimconnector(
 436                                 uuid=vim['datacenter_id'], name=vim['datacenter_name'],
 437                                 tenant_id=vim.get('vim_tenant_id',vim_tenant),
 438                                 tenant_name=vim.get('vim_tenant_name',vim_tenant_name),
 439                                 url=vim['vim_url'], url_admin=vim['vim_url_admin'],
 440                                 user=vim.get('user',vim_user), passwd=vim.get('passwd',vim_passwd),
 441                                 config=extra, persistent_info=persistent_info
 442                         )
 443             except Exception as e:
 444                 if ignore_errors:
 445                     logger.error("Error at VIM  {}; {}: {}".format(vim["type"], type(e).__name__, str(e)))
 446                     continue
 447                 http_code = httperrors.Internal_Server_Error
 448                 if isinstance(e, vimconn.vimconnException):
 449                     http_code = e.http_code
 450                 raise NfvoException("Error at VIM  {}; {}: {}".format(vim["type"], type(e).__name__, str(e)), http_code)
 451         return vim_dict
 452     except db_base_Exception as e:
 453         raise NfvoException(str(e) + " at nfvo.get_vim", e.http_code)
 454
 455
 456 def rollback(mydb,  vims, rollback_list):
 457     undeleted_items=[]
 458     #delete things by reverse order
 459     for i in range(len(rollback_list)-1, -1, -1):
 460         item = rollback_list[i]
 461         if item["where"]=="vim":
 462             if item["vim_id"] not in vims:
 463                 continue
 464             if is_task_id(item["uuid"]):
 465                 continue
 466             vim = vims[item["vim_id"]]
 467             try:
 468                 if item["what"]=="image":
 469                     vim.delete_image(item["uuid"])
 470                     mydb.delete_row(FROM="datacenters_images", WHERE={"datacenter_vim_id": vim["id"], "vim_id":item["uuid"]})
 471                 elif item["what"]=="flavor":
 472                     vim.delete_flavor(item["uuid"])
 473                     mydb.delete_row(FROM="datacenters_flavors", WHERE={"datacenter_vim_id": vim["id"], "vim_id":item["uuid"]})
 474                 elif item["what"]=="network":
 475                     vim.delete_network(item["uuid"])
 476                 elif item["what"]=="vm":
 477                     vim.delete_vminstance(item["uuid"])
 478             except vimconn.vimconnException as e:
 479                 logger.error("Error in rollback. Not possible to delete VIM %s '%s'. Message: %s", item['what'], item["uuid"], str(e))
 480                 undeleted_items.append("{} {} from VIM {}".format(item['what'], item["uuid"], vim["name"]))
 481             except db_base_Exception as e:
 482                 logger.error("Error in rollback. Not possible to delete %s '%s' from DB.datacenters Message: %s", item['what'], item["uuid"], str(e))
 483
 484         else: # where==mano
 485             try:
 486                 if item["what"]=="image":
 487                     mydb.delete_row(FROM="images", WHERE={"uuid": item["uuid"]})
 488                 elif item["what"]=="flavor":
 489                     mydb.delete_row(FROM="flavors", WHERE={"uuid": item["uuid"]})
 490             except db_base_Exception as e:
 491                 logger.error("Error in rollback. Not possible to delete %s '%s' from DB. Message: %s", item['what'], item["uuid"], str(e))
 492                 undeleted_items.append("{} '{}'".format(item['what'], item["uuid"]))
 493     if len(undeleted_items)==0:
 494         return True," Rollback successful."
 495     else:
 496         return False," Rollback fails to delete: " + str(undeleted_items)
 497
 498
 499 def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1):
 500     global global_config
 501     #create a dictionary with vnfc-name: vnfc:interface-list  key:values pairs
 502     vnfc_interfaces={}
 503     for vnfc in vnf_descriptor["vnf"]["VNFC"]:
 504         name_dict = {}
 505         #dataplane interfaces
 506         for numa in vnfc.get("numas",() ):
 507             for interface in numa.get("interfaces",()):
 508                 if interface["name"] in name_dict:
 509                     raise NfvoException(
 510                         "Error at vnf:VNFC[name:'{}']:numas:interfaces:name, interface name '{}' already used in this VNFC".format(
 511                             vnfc["name"], interface["name"]),
 512                         httperrors.Bad_Request)
 513                 name_dict[ interface["name"] ] = "underlay"
 514         #bridge interfaces
 515         for interface in vnfc.get("bridge-ifaces",() ):
 516             if interface["name"] in name_dict:
 517                 raise NfvoException(
 518                     "Error at vnf:VNFC[name:'{}']:bridge-ifaces:name, interface name '{}' already used in this VNFC".format(
 519                         vnfc["name"], interface["name"]),
 520                     httperrors.Bad_Request)
 521             name_dict[ interface["name"] ] = "overlay"
 522         vnfc_interfaces[ vnfc["name"] ] = name_dict
 523         # check bood-data info
 524         # if "boot-data" in vnfc:
 525         #     # check that user-data is incompatible with users and config-files
 526         #     if (vnfc["boot-data"].get("users") or vnfc["boot-data"].get("config-files")) and vnfc["boot-data"].get("user-data"):
 527         #         raise NfvoException(
 528         #             "Error at vnf:VNFC:boot-data, fields 'users' and 'config-files' are not compatible with 'user-data'",
 529         #             httperrors.Bad_Request)
 530
 531     #check if the info in external_connections matches with the one in the vnfcs
 532     name_list=[]
 533     for external_connection in vnf_descriptor["vnf"].get("external-connections",() ):
 534         if external_connection["name"] in name_list:
 535             raise NfvoException(
 536                 "Error at vnf:external-connections:name, value '{}' already used as an external-connection".format(
 537                     external_connection["name"]),
 538                 httperrors.Bad_Request)
 539         name_list.append(external_connection["name"])
 540         if external_connection["VNFC"] not in vnfc_interfaces:
 541             raise NfvoException(
 542                 "Error at vnf:external-connections[name:'{}']:VNFC, value '{}' does not match any VNFC".format(
 543                     external_connection["name"], external_connection["VNFC"]),
 544                 httperrors.Bad_Request)
 545
 546         if external_connection["local_iface_name"] not in vnfc_interfaces[ external_connection["VNFC"] ]:
 547             raise NfvoException(
 548                 "Error at vnf:external-connections[name:'{}']:local_iface_name, value '{}' does not match any interface of this VNFC".format(
 549                     external_connection["name"],
 550                     external_connection["local_iface_name"]),
 551                 httperrors.Bad_Request )
 552
 553     #check if the info in internal_connections matches with the one in the vnfcs
 554     name_list=[]
 555     for internal_connection in vnf_descriptor["vnf"].get("internal-connections",() ):
 556         if internal_connection["name"] in name_list:
 557             raise NfvoException(
 558                 "Error at vnf:internal-connections:name, value '{}' already used as an internal-connection".format(
 559                     internal_connection["name"]),
 560                 httperrors.Bad_Request)
 561         name_list.append(internal_connection["name"])
 562         #We should check that internal-connections of type "ptp" have only 2 elements
 563
 564         if len(internal_connection["elements"])>2 and (internal_connection.get("type") == "ptp" or internal_connection.get("type") == "e-line"):
 565             raise NfvoException(
 566                 "Error at 'vnf:internal-connections[name:'{}']:elements', size must be 2 for a '{}' type. Consider change it to '{}' type".format(
 567                     internal_connection["name"],
 568                     'ptp' if vnf_descriptor_version==1 else 'e-line',
 569                     'data' if vnf_descriptor_version==1 else "e-lan"),
 570                 httperrors.Bad_Request)
 571         for port in internal_connection["elements"]:
 572             vnf = port["VNFC"]
 573             iface = port["local_iface_name"]
 574             if vnf not in vnfc_interfaces:
 575                 raise NfvoException(
 576                     "Error at vnf:internal-connections[name:'{}']:elements[]:VNFC, value '{}' does not match any VNFC".format(
 577                         internal_connection["name"], vnf),
 578                     httperrors.Bad_Request)
 579             if iface not in vnfc_interfaces[ vnf ]:
 580                 raise NfvoException(
 581                     "Error at vnf:internal-connections[name:'{}']:elements[]:local_iface_name, value '{}' does not match any interface of this VNFC".format(
 582                         internal_connection["name"], iface),
 583                     httperrors.Bad_Request)
 584                 return -httperrors.Bad_Request,
 585             if vnf_descriptor_version==1 and "type" not in internal_connection:
 586                 if vnfc_interfaces[vnf][iface] == "overlay":
 587                     internal_connection["type"] = "bridge"
 588                 else:
 589                     internal_connection["type"] = "data"
 590             if vnf_descriptor_version==2 and "implementation" not in internal_connection:
 591                 if vnfc_interfaces[vnf][iface] == "overlay":
 592                     internal_connection["implementation"] = "overlay"
 593                 else:
 594                     internal_connection["implementation"] = "underlay"
 595             if (internal_connection.get("type") == "data" or internal_connection.get("type") == "ptp" or \
 596                 internal_connection.get("implementation") == "underlay") and vnfc_interfaces[vnf][iface] == "overlay":
 597                 raise NfvoException(
 598                     "Error at vnf:internal-connections[name:'{}']:elements[]:{}, interface of type {} connected to an {} network".format(
 599                         internal_connection["name"],
 600                         iface, 'bridge' if vnf_descriptor_version==1 else 'overlay',
 601                         'data' if vnf_descriptor_version==1 else 'underlay'),
 602                     httperrors.Bad_Request)
 603             if (internal_connection.get("type") == "bridge" or internal_connection.get("implementation") == "overlay") and \
 604                 vnfc_interfaces[vnf][iface] == "underlay":
 605                 raise NfvoException(
 606                     "Error at vnf:internal-connections[name:'{}']:elements[]:{}, interface of type {} connected to an {} network".format(
 607                         internal_connection["name"], iface,
 608                         'data' if vnf_descriptor_version==1 else 'underlay',
 609                         'bridge' if vnf_descriptor_version==1 else 'overlay'),
 610                     httperrors.Bad_Request)
 611
 612
 613 def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=False, return_on_error=None):
 614     #look if image exist
 615     if only_create_at_vim:
 616         image_mano_id = image_dict['uuid']
 617         if return_on_error == None:
 618             return_on_error = True
 619     else:
 620         if image_dict['location']:
 621             images = mydb.get_rows(FROM="images", WHERE={'location':image_dict['location'], 'metadata':image_dict['metadata']})
 622         else:
 623             images = mydb.get_rows(FROM="images", WHERE={'universal_name':image_dict['universal_name'], 'checksum':image_dict['checksum']})
 624         if len(images)>=1:
 625             image_mano_id = images[0]['uuid']
 626         else:
 627             #create image in MANO DB
 628             temp_image_dict={'name':image_dict['name'],         'description':image_dict.get('description',None),
 629                             'location':image_dict['location'],  'metadata':image_dict.get('metadata',None),
 630                             'universal_name':image_dict['universal_name'] , 'checksum':image_dict['checksum']
 631                             }
 632             #temp_image_dict['location'] = image_dict.get('new_location') if image_dict['location'] is None
 633             image_mano_id = mydb.new_row('images', temp_image_dict, add_uuid=True)
 634             rollback_list.append({"where":"mano", "what":"image","uuid":image_mano_id})
 635     #create image at every vim
 636     for vim_id,vim in vims.items():
 637         datacenter_vim_id = vim["config"]["datacenter_tenant_id"]
 638         image_created="false"
 639         #look at database
 640         image_db = mydb.get_rows(FROM="datacenters_images",
 641                                  WHERE={'datacenter_vim_id': datacenter_vim_id, 'image_id': image_mano_id})
 642         #look at VIM if this image exist
 643         try:
 644             if image_dict['location'] is not None:
 645                 image_vim_id = vim.get_image_id_from_path(image_dict['location'])
 646             else:
 647                 filter_dict = {}
 648                 filter_dict['name'] = image_dict['universal_name']
 649                 if image_dict.get('checksum') != None:
 650                     filter_dict['checksum'] = image_dict['checksum']
 651                 #logger.debug('>>>>>>>> Filter dict: %s', str(filter_dict))
 652                 vim_images = vim.get_image_list(filter_dict)
 653                 #logger.debug('>>>>>>>> VIM images: %s', str(vim_images))
 654                 if len(vim_images) > 1:
 655                     raise vimconn.vimconnException("More than one candidate VIM image found for filter: {}".format(str(filter_dict)), httperrors.Conflict)
 656                 elif len(vim_images) == 0:
 657                     raise vimconn.vimconnNotFoundException("Image not found at VIM with filter: '{}'".format(str(filter_dict)))
 658                 else:
 659                     #logger.debug('>>>>>>>> VIM image 0: %s', str(vim_images[0]))
 660                     image_vim_id = vim_images[0]['id']
 661
 662         except vimconn.vimconnNotFoundException as e:
 663             #Create the image in VIM only if image_dict['location'] or image_dict['new_location'] is not None
 664             try:
 665                 #image_dict['location']=image_dict.get('new_location') if image_dict['location'] is None
 666                 if image_dict['location']:
 667                     image_vim_id = vim.new_image(image_dict)
 668                     rollback_list.append({"where":"vim", "vim_id": vim_id, "what":"image","uuid":image_vim_id})
 669                     image_created="true"
 670                 else:
 671                     #If we reach this point, then the image has image name, and optionally checksum, and could not be found
 672                     raise vimconn.vimconnException(str(e))
 673             except vimconn.vimconnException as e:
 674                 if return_on_error:
 675                     logger.error("Error creating image at VIM '%s': %s", vim["name"], str(e))
 676                     raise
 677                 image_vim_id = None
 678                 logger.warn("Error creating image at VIM '%s': %s", vim["name"], str(e))
 679                 continue
 680         except vimconn.vimconnException as e:
 681             if return_on_error:
 682                 logger.error("Error contacting VIM to know if the image exists at VIM: %s", str(e))
 683                 raise
 684             logger.warn("Error contacting VIM to know if the image exists at VIM: %s", str(e))
 685             image_vim_id = None
 686             continue
 687         #if we reach here, the image has been created or existed
 688         if len(image_db)==0:
 689             #add new vim_id at datacenters_images
 690             mydb.new_row('datacenters_images', {'datacenter_vim_id': datacenter_vim_id,
 691                                                 'image_id':image_mano_id,
 692                                                 'vim_id': image_vim_id,
 693                                                 'created':image_created})
 694         elif image_db[0]["vim_id"]!=image_vim_id:
 695             #modify existing vim_id at datacenters_images
 696             mydb.update_rows('datacenters_images', UPDATE={'vim_id':image_vim_id}, WHERE={'datacenter_vim_id':vim_id, 'image_id':image_mano_id})
 697
 698     return image_vim_id if only_create_at_vim else image_mano_id
 699
 700
 701 def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_vim=False, return_on_error = None):
 702     temp_flavor_dict= {'disk':flavor_dict.get('disk',0),
 703             'ram':flavor_dict.get('ram'),
 704             'vcpus':flavor_dict.get('vcpus'),
 705         }
 706     if 'extended' in flavor_dict and flavor_dict['extended']==None:
 707         del flavor_dict['extended']
 708     if 'extended' in flavor_dict:
 709         temp_flavor_dict['extended']=yaml.safe_dump(flavor_dict['extended'],default_flow_style=True,width=256)
 710
 711     #look if flavor exist
 712     if only_create_at_vim:
 713         flavor_mano_id = flavor_dict['uuid']
 714         if return_on_error == None:
 715             return_on_error = True
 716     else:
 717         flavors = mydb.get_rows(FROM="flavors", WHERE=temp_flavor_dict)
 718         if len(flavors)>=1:
 719             flavor_mano_id = flavors[0]['uuid']
 720         else:
 721             #create flavor
 722             #create one by one the images of aditional disks
 723             dev_image_list=[] #list of images
 724             if 'extended' in flavor_dict and flavor_dict['extended']!=None:
 725                 dev_nb=0
 726                 for device in flavor_dict['extended'].get('devices',[]):
 727                     if "image" not in device and "image name" not in device:
 728                         continue
 729                     image_dict={}
 730                     image_dict['name']=device.get('image name',flavor_dict['name']+str(dev_nb)+"-img")
 731                     image_dict['universal_name']=device.get('image name')
 732                     image_dict['description']=flavor_dict['name']+str(dev_nb)+"-img"
 733                     image_dict['location']=device.get('image')
 734                     #image_dict['new_location']=vnfc.get('image location')
 735                     image_dict['checksum']=device.get('image checksum')
 736                     image_metadata_dict = device.get('image metadata', None)
 737                     image_metadata_str = None
 738                     if image_metadata_dict != None:
 739                         image_metadata_str = yaml.safe_dump(image_metadata_dict,default_flow_style=True,width=256)
 740                     image_dict['metadata']=image_metadata_str
 741                     image_id = create_or_use_image(mydb, vims, image_dict, rollback_list)
 742                     #print "Additional disk image id for VNFC %s: %s" % (flavor_dict['name']+str(dev_nb)+"-img", image_id)
 743                     dev_image_list.append(image_id)
 744                     dev_nb += 1
 745             temp_flavor_dict['name'] = flavor_dict['name']
 746             temp_flavor_dict['description'] = flavor_dict.get('description',None)
 747             content = mydb.new_row('flavors', temp_flavor_dict, add_uuid=True)
 748             flavor_mano_id= content
 749             rollback_list.append({"where":"mano", "what":"flavor","uuid":flavor_mano_id})
 750     #create flavor at every vim
 751     if 'uuid' in flavor_dict:
 752         del flavor_dict['uuid']
 753     flavor_vim_id=None
 754     for vim_id,vim in vims.items():
 755         datacenter_vim_id = vim["config"]["datacenter_tenant_id"]
 756         flavor_created="false"
 757         #look at database
 758         flavor_db = mydb.get_rows(FROM="datacenters_flavors",
 759                                   WHERE={'datacenter_vim_id': datacenter_vim_id, 'flavor_id': flavor_mano_id})
 760         #look at VIM if this flavor exist  SKIPPED
 761         #res_vim, flavor_vim_id = vim.get_flavor_id_from_path(flavor_dict['location'])
 762         #if res_vim < 0:
 763         #    print "Error contacting VIM to know if the flavor %s existed previously." %flavor_vim_id
 764         #    continue
 765         #elif res_vim==0:
 766
 767         # Create the flavor in VIM
 768         # Translate images at devices from MANO id to VIM id
 769         disk_list = []
 770         if 'extended' in flavor_dict and flavor_dict['extended']!=None and "devices" in flavor_dict['extended']:
 771             # make a copy of original devices
 772             devices_original=[]
 773
 774             for device in flavor_dict["extended"].get("devices",[]):
 775                 dev={}
 776                 dev.update(device)
 777                 devices_original.append(dev)
 778                 if 'image' in device:
 779                     del device['image']
 780                 if 'image metadata' in device:
 781                     del device['image metadata']
 782                 if 'image checksum' in device:
 783                     del device['image checksum']
 784             dev_nb = 0
 785             for index in range(0,len(devices_original)) :
 786                 device=devices_original[index]
 787                 if "image" not in device and "image name" not in device:
 788                     # if 'size' in device:
 789                     disk_list.append({'size': device.get('size', default_volume_size), 'name': device.get('name')})
 790                     continue
 791                 image_dict={}
 792                 image_dict['name']=device.get('image name',flavor_dict['name']+str(dev_nb)+"-img")
 793                 image_dict['universal_name']=device.get('image name')
 794                 image_dict['description']=flavor_dict['name']+str(dev_nb)+"-img"
 795                 image_dict['location']=device.get('image')
 796                 # image_dict['new_location']=device.get('image location')
 797                 image_dict['checksum']=device.get('image checksum')
 798                 image_metadata_dict = device.get('image metadata', None)
 799                 image_metadata_str = None
 800                 if image_metadata_dict != None:
 801                     image_metadata_str = yaml.safe_dump(image_metadata_dict,default_flow_style=True,width=256)
 802                 image_dict['metadata']=image_metadata_str
 803                 image_mano_id=create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=False, return_on_error=return_on_error )
 804                 image_dict["uuid"]=image_mano_id
 805                 image_vim_id=create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=True, return_on_error=return_on_error)
 806
 807                 #save disk information (image must be based on and size
 808                 disk_list.append({'image_id': image_vim_id, 'size': device.get('size', default_volume_size)})
 809
 810                 flavor_dict["extended"]["devices"][index]['imageRef']=image_vim_id
 811                 dev_nb += 1
 812         if len(flavor_db)>0:
 813             #check that this vim_id exist in VIM, if not create
 814             flavor_vim_id=flavor_db[0]["vim_id"]
 815             try:
 816                 vim.get_flavor(flavor_vim_id)
 817                 continue #flavor exist
 818             except vimconn.vimconnException:
 819                 pass
 820         #create flavor at vim
 821         logger.debug("nfvo.create_or_use_flavor() adding flavor to VIM %s", vim["name"])
 822         try:
 823             flavor_vim_id = None
 824             flavor_vim_id=vim.get_flavor_id_from_data(flavor_dict)
 825             flavor_created="false"
 826         except vimconn.vimconnException as e:
 827             pass
 828         try:
 829             if not flavor_vim_id:
 830                 flavor_vim_id = vim.new_flavor(flavor_dict)
 831                 rollback_list.append({"where":"vim", "vim_id": vim_id, "what":"flavor","uuid":flavor_vim_id})
 832                 flavor_created="true"
 833         except vimconn.vimconnException as e:
 834             if return_on_error:
 835                 logger.error("Error creating flavor at VIM %s: %s.", vim["name"], str(e))
 836                 raise
 837             logger.warn("Error creating flavor at VIM %s: %s.", vim["name"], str(e))
 838             flavor_vim_id = None
 839             continue
 840         #if reach here the flavor has been create or exist
 841         if len(flavor_db)==0:
 842             #add new vim_id at datacenters_flavors
 843             extended_devices_yaml = None
 844             if len(disk_list) > 0:
 845                 extended_devices = dict()
 846                 extended_devices['disks'] = disk_list
 847                 extended_devices_yaml = yaml.safe_dump(extended_devices,default_flow_style=True,width=256)
 848             mydb.new_row('datacenters_flavors',
 849                         {'datacenter_vim_id': datacenter_vim_id, 'flavor_id': flavor_mano_id, 'vim_id': flavor_vim_id,
 850                         'created': flavor_created, 'extended': extended_devices_yaml})
 851         elif flavor_db[0]["vim_id"]!=flavor_vim_id:
 852             #modify existing vim_id at datacenters_flavors
 853             mydb.update_rows('datacenters_flavors', UPDATE={'vim_id':flavor_vim_id},
 854                              WHERE={'datacenter_vim_id': datacenter_vim_id, 'flavor_id': flavor_mano_id})
 855
 856     return flavor_vim_id if only_create_at_vim else flavor_mano_id
 857
 858
 859 def get_str(obj, field, length):
 860     """
 861     Obtain the str value,
 862     :param obj:
 863     :param length:
 864     :return:
 865     """
 866     value = obj.get(field)
 867     if value is not None:
 868         value = str(value)[:length]
 869     return value
 870
 871 def _lookfor_or_create_image(db_image, mydb, descriptor):
 872     """
 873     fill image content at db_image dictionary. Check if the image with this image and checksum exist
 874     :param db_image: dictionary to insert data
 875     :param mydb: database connector
 876     :param descriptor: yang descriptor
 877     :return: uuid if the image exist at DB, or None if a new image must be created with the data filled at db_image
 878     """
 879
 880     db_image["name"] = get_str(descriptor, "image", 255)
 881     db_image["checksum"] = get_str(descriptor, "image-checksum", 32)
 882     if not db_image["checksum"]:  # Ensure that if empty string, None is stored
 883         db_image["checksum"] = None
 884     if db_image["name"].startswith("/"):
 885         db_image["location"] = db_image["name"]
 886         existing_images = mydb.get_rows(FROM="images", WHERE={'location': db_image["location"]})
 887     else:
 888         db_image["universal_name"] = db_image["name"]
 889         existing_images = mydb.get_rows(FROM="images", WHERE={'universal_name': db_image['universal_name'],
 890                                                               'checksum': db_image['checksum']})
 891     if existing_images:
 892         return existing_images[0]["uuid"]
 893     else:
 894         image_uuid = str(uuid4())
 895         db_image["uuid"] = image_uuid
 896         return None
 897
 898 def get_resource_allocation_params(quota_descriptor):
 899     """
 900     read the quota_descriptor from vnfd and fetch the resource allocation properties from the descriptor object
 901     :param quota_descriptor: cpu/mem/vif/disk-io quota descriptor
 902     :return: quota params for limit, reserve, shares from the descriptor object
 903     """
 904     quota = {}
 905     if quota_descriptor.get("limit"):
 906         quota["limit"] = int(quota_descriptor["limit"])
 907     if quota_descriptor.get("reserve"):
 908         quota["reserve"] = int(quota_descriptor["reserve"])
 909     if quota_descriptor.get("shares"):
 910         quota["shares"] = int(quota_descriptor["shares"])
 911     return quota
 912
 913 def new_vnfd_v3(mydb, tenant_id, vnf_descriptor):
 914     """
 915     Parses an OSM IM vnfd_catalog and insert at DB
 916     :param mydb:
 917     :param tenant_id:
 918     :param vnf_descriptor:
 919     :return: The list of cretated vnf ids
 920     """
 921     try:
 922         myvnfd = vnfd_catalog.vnfd()
 923         try:
 924             pybindJSONDecoder.load_ietf_json(vnf_descriptor, None, None, obj=myvnfd, path_helper=True,
 925                                              skip_unknown=True)
 926         except Exception as e:
 927             raise NfvoException("Error. Invalid VNF descriptor format " + str(e), httperrors.Bad_Request)
 928         db_vnfs = []
 929         db_nets = []
 930         db_vms = []
 931         db_vms_index = 0
 932         db_interfaces = []
 933         db_images = []
 934         db_flavors = []
 935         db_ip_profiles_index = 0
 936         db_ip_profiles = []
 937         uuid_list = []
 938         vnfd_uuid_list = []
 939         vnfd_catalog_descriptor = vnf_descriptor.get("vnfd:vnfd-catalog")
 940         if not vnfd_catalog_descriptor:
 941             vnfd_catalog_descriptor = vnf_descriptor.get("vnfd-catalog")
 942         vnfd_descriptor_list = vnfd_catalog_descriptor.get("vnfd")
 943         if not vnfd_descriptor_list:
 944             vnfd_descriptor_list = vnfd_catalog_descriptor.get("vnfd:vnfd")
 945         for vnfd_yang in myvnfd.vnfd_catalog.vnfd.values():
 946             vnfd = vnfd_yang.get()
 947
 948             # table vnf
 949             vnf_uuid = str(uuid4())
 950             uuid_list.append(vnf_uuid)
 951             vnfd_uuid_list.append(vnf_uuid)
 952             vnfd_id = get_str(vnfd, "id", 255)
 953             db_vnf = {
 954                 "uuid": vnf_uuid,
 955                 "osm_id": vnfd_id,
 956                 "name": get_str(vnfd, "name", 255),
 957                 "description": get_str(vnfd, "description", 255),
 958                 "tenant_id": tenant_id,
 959                 "vendor": get_str(vnfd, "vendor", 255),
 960                 "short_name": get_str(vnfd, "short-name", 255),
 961                 "descriptor": str(vnf_descriptor)[:60000]
 962             }
 963
 964             for vnfd_descriptor in vnfd_descriptor_list:
 965                 if vnfd_descriptor["id"] == str(vnfd["id"]):
 966                     break
 967
 968             # table ip_profiles (ip-profiles)
 969             ip_profile_name2db_table_index = {}
 970             for ip_profile in vnfd.get("ip-profiles").values():
 971                 db_ip_profile = {
 972                     "ip_version": str(ip_profile["ip-profile-params"].get("ip-version", "ipv4")),
 973                     "subnet_address": str(ip_profile["ip-profile-params"].get("subnet-address")),
 974                     "gateway_address": str(ip_profile["ip-profile-params"].get("gateway-address")),
 975                     "dhcp_enabled": str(ip_profile["ip-profile-params"]["dhcp-params"].get("enabled", True)),
 976                     "dhcp_start_address": str(ip_profile["ip-profile-params"]["dhcp-params"].get("start-address")),
 977                     "dhcp_count": str(ip_profile["ip-profile-params"]["dhcp-params"].get("count")),
 978                 }
 979                 dns_list = []
 980                 for dns in ip_profile["ip-profile-params"]["dns-server"].values():
 981                     dns_list.append(str(dns.get("address")))
 982                 db_ip_profile["dns_address"] = ";".join(dns_list)
 983                 if ip_profile["ip-profile-params"].get('security-group'):
 984                     db_ip_profile["security_group"] = ip_profile["ip-profile-params"]['security-group']
 985                 ip_profile_name2db_table_index[str(ip_profile["name"])] = db_ip_profiles_index
 986                 db_ip_profiles_index += 1
 987                 db_ip_profiles.append(db_ip_profile)
 988
 989             # table nets (internal-vld)
 990             net_id2uuid = {}  # for mapping interface with network
 991             net_id2index = {}  # for mapping interface with network
 992             for vld in vnfd.get("internal-vld").values():
 993                 net_uuid = str(uuid4())
 994                 uuid_list.append(net_uuid)
 995                 db_net = {
 996                     "name": get_str(vld, "name", 255),
 997                     "vnf_id": vnf_uuid,
 998                     "uuid": net_uuid,
 999                     "description": get_str(vld, "description", 255),
1000                     "osm_id": get_str(vld, "id", 255),
1001                     "type": "bridge",   # TODO adjust depending on connection point type
1002                 }
1003                 net_id2uuid[vld.get("id")] = net_uuid
1004                 net_id2index[vld.get("id")] = len(db_nets)
1005                 db_nets.append(db_net)
1006                 # ip-profile, link db_ip_profile with db_sce_net
1007                 if vld.get("ip-profile-ref"):
1008                     ip_profile_name = vld.get("ip-profile-ref")
1009                     if ip_profile_name not in ip_profile_name2db_table_index:
1010                         raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{}]':'vld[{}]':'ip-profile-ref':"
1011                                             "'{}'. Reference to a non-existing 'ip_profiles'".format(
1012                                                 str(vnfd["id"]), str(vld["id"]), str(vld["ip-profile-ref"])),
1013                                             httperrors.Bad_Request)
1014                     db_ip_profiles[ip_profile_name2db_table_index[ip_profile_name]]["net_id"] = net_uuid
1015                 else:  #check no ip-address has been defined
1016                     for icp in vld.get("internal-connection-point").values():
1017                         if icp.get("ip-address"):
1018                             raise NfvoException("Error at 'vnfd[{}]':'vld[{}]':'internal-connection-point[{}]' "
1019                                             "contains an ip-address but no ip-profile has been defined at VLD".format(
1020                                                 str(vnfd["id"]), str(vld["id"]), str(icp["id"])),
1021                                             httperrors.Bad_Request)
1022
1023             # connection points vaiable declaration
1024             cp_name2iface_uuid = {}
1025             cp_name2vdu_id = {}
1026             cp_name2vm_uuid = {}
1027             cp_name2db_interface = {}
1028             vdu_id2cp_name = {}  # stored only when one external connection point is presented at this VDU
1029
1030             # table vms (vdus)
1031             vdu_id2uuid = {}
1032             vdu_id2db_table_index = {}
1033             mgmt_access = {}
1034             for vdu in vnfd.get("vdu").values():
1035
1036                 for vdu_descriptor in vnfd_descriptor["vdu"]:
1037                     if vdu_descriptor["id"] == str(vdu["id"]):
1038                         break
1039                 vm_uuid = str(uuid4())
1040                 uuid_list.append(vm_uuid)
1041                 vdu_id = get_str(vdu, "id", 255)
1042                 db_vm = {
1043                     "uuid": vm_uuid,
1044                     "osm_id": vdu_id,
1045                     "name": get_str(vdu, "name", 255),
1046                     "description": get_str(vdu, "description", 255),
1047                     "pdu_type": get_str(vdu, "pdu-type", 255),
1048                     "vnf_id": vnf_uuid,
1049                 }
1050                 vdu_id2uuid[db_vm["osm_id"]] = vm_uuid
1051                 vdu_id2db_table_index[db_vm["osm_id"]] = db_vms_index
1052                 if vdu.get("count"):
1053                     db_vm["count"] = int(vdu["count"])
1054
1055                 # table image
1056                 image_present = False
1057                 if vdu.get("image"):
1058                     image_present = True
1059                     db_image = {}
1060                     image_uuid = _lookfor_or_create_image(db_image, mydb, vdu)
1061                     if not image_uuid:
1062                         image_uuid = db_image["uuid"]
1063                         db_images.append(db_image)
1064                     db_vm["image_id"] = image_uuid
1065                 if vdu.get("alternative-images"):
1066                     vm_alternative_images = []
1067                     for alt_image in vdu.get("alternative-images").values():
1068                         db_image = {}
1069                         image_uuid = _lookfor_or_create_image(db_image, mydb, alt_image)
1070                         if not image_uuid:
1071                             image_uuid = db_image["uuid"]
1072                             db_images.append(db_image)
1073                         vm_alternative_images.append({
1074                             "image_id": image_uuid,
1075                             "vim_type": str(alt_image["vim-type"]),
1076                             # "universal_name": str(alt_image["image"]),
1077                             # "checksum": str(alt_image["image-checksum"]) if alt_image.get("image-checksum") else None
1078                         })
1079
1080                     db_vm["image_list"] = yaml.safe_dump(vm_alternative_images, default_flow_style=True, width=256)
1081
1082                 # volumes
1083                 devices = []
1084                 if vdu.get("volumes"):
1085                     for volume_key in vdu["volumes"]:
1086                         volume = vdu["volumes"][volume_key]
1087                         if not image_present:
1088                             # Convert the first volume to vnfc.image
1089                             image_present = True
1090                             db_image = {}
1091                             image_uuid = _lookfor_or_create_image(db_image, mydb, volume)
1092                             if not image_uuid:
1093                                 image_uuid = db_image["uuid"]
1094                                 db_images.append(db_image)
1095                             db_vm["image_id"] = image_uuid
1096                         else:
1097                             # Add Openmano devices
1098                             device = {"name": str(volume.get("name"))}
1099                             device["type"] = str(volume.get("device-type"))
1100                             if volume.get("size"):
1101                                 device["size"] = int(volume["size"])
1102                             if volume.get("image"):
1103                                 device["image name"] = str(volume["image"])
1104                                 if volume.get("image-checksum"):
1105                                     device["image checksum"] = str(volume["image-checksum"])
1106
1107                             devices.append(device)
1108
1109                 if not db_vm.get("image_id"):
1110                     if not db_vm["pdu_type"]:
1111                         raise NfvoException("Not defined image for VDU")
1112                     # create a fake image
1113
1114                 # cloud-init
1115                 boot_data = {}
1116                 if vdu.get("cloud-init"):
1117                     boot_data["user-data"] = str(vdu["cloud-init"])
1118                 elif vdu.get("cloud-init-file"):
1119                     # TODO Where this file content is present???
1120                     # boot_data["user-data"] = vnfd_yang.files[vdu["cloud-init-file"]]
1121                     boot_data["user-data"] = str(vdu["cloud-init-file"])
1122
1123                 if vdu.get("supplemental-boot-data"):
1124                     if vdu["supplemental-boot-data"].get('boot-data-drive'):
1125                             boot_data['boot-data-drive'] = True
1126                     if vdu["supplemental-boot-data"].get('config-file'):
1127                         om_cfgfile_list = list()
1128                         for custom_config_file in vdu["supplemental-boot-data"]['config-file'].values():
1129                             # TODO Where this file content is present???
1130                             cfg_source = str(custom_config_file["source"])
1131                             om_cfgfile_list.append({"dest": custom_config_file["dest"],
1132                                                     "content": cfg_source})
1133                         boot_data['config-files'] = om_cfgfile_list
1134                 if boot_data:
1135                     db_vm["boot_data"] = yaml.safe_dump(boot_data, default_flow_style=True, width=256)
1136
1137                 db_vms.append(db_vm)
1138                 db_vms_index += 1
1139
1140                 # table interfaces (internal/external interfaces)
1141                 flavor_epa_interfaces = []
1142                 # for iface in chain(vdu.get("internal-interface").values(), vdu.get("external-interface").values()):
1143                 for iface in vdu.get("interface").values():
1144                     flavor_epa_interface = {}
1145                     iface_uuid = str(uuid4())
1146                     uuid_list.append(iface_uuid)
1147                     db_interface = {
1148                         "uuid": iface_uuid,
1149                         "internal_name": get_str(iface, "name", 255),
1150                         "vm_id": vm_uuid,
1151                     }
1152                     flavor_epa_interface["name"] = db_interface["internal_name"]
1153                     if iface.get("virtual-interface").get("vpci"):
1154                         db_interface["vpci"] = get_str(iface.get("virtual-interface"), "vpci", 12)
1155                         flavor_epa_interface["vpci"] = db_interface["vpci"]
1156
1157                     if iface.get("virtual-interface").get("bandwidth"):
1158                         bps = int(iface.get("virtual-interface").get("bandwidth"))
1159                         db_interface["bw"] = int(math.ceil(bps / 1000000.0))
1160                         flavor_epa_interface["bandwidth"] = "{} Mbps".format(db_interface["bw"])
1161
1162                     if iface.get("virtual-interface").get("type") == "OM-MGMT":
1163                         db_interface["type"] = "mgmt"
1164                     elif iface.get("virtual-interface").get("type") in ("VIRTIO", "E1000", "PARAVIRT"):
1165                         db_interface["type"] = "bridge"
1166                         db_interface["model"] = get_str(iface.get("virtual-interface"), "type", 12)
1167                     elif iface.get("virtual-interface").get("type") in ("SR-IOV", "PCI-PASSTHROUGH"):
1168                         db_interface["type"] = "data"
1169                         db_interface["model"] = get_str(iface.get("virtual-interface"), "type", 12)
1170                         flavor_epa_interface["dedicated"] = "no" if iface["virtual-interface"]["type"] == "SR-IOV" \
1171                             else "yes"
1172                         flavor_epa_interfaces.append(flavor_epa_interface)
1173                     else:
1174                         raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{}]':'vdu[{}]':'interface':'virtual"
1175                                             "-interface':'type':'{}'. Interface type is not supported".format(
1176                                                 vnfd_id, vdu_id, iface.get("virtual-interface").get("type")),
1177                                             httperrors.Bad_Request)
1178
1179                     if iface.get("mgmt-interface"):
1180                         db_interface["type"] = "mgmt"
1181
1182                     if iface.get("external-connection-point-ref"):
1183                         try:
1184                             cp = vnfd.get("connection-point")[iface.get("external-connection-point-ref")]
1185                             db_interface["external_name"] = get_str(cp, "name", 255)
1186                             cp_name2iface_uuid[db_interface["external_name"]] = iface_uuid
1187                             cp_name2vdu_id[db_interface["external_name"]] = vdu_id
1188                             cp_name2vm_uuid[db_interface["external_name"]] = vm_uuid
1189                             cp_name2db_interface[db_interface["external_name"]] = db_interface
1190                             for cp_descriptor in vnfd_descriptor["connection-point"]:
1191                                 if cp_descriptor["name"] == db_interface["external_name"]:
1192                                     break
1193                             else:
1194                                 raise KeyError()
1195
1196                             if vdu_id in vdu_id2cp_name:
1197                                 vdu_id2cp_name[vdu_id] = None  # more than two connection point for this VDU
1198                             else:
1199                                 vdu_id2cp_name[vdu_id] = db_interface["external_name"]
1200
1201                             # port security
1202                             if str(cp_descriptor.get("port-security-enabled")).lower() == "false":
1203                                 db_interface["port_security"] = 0
1204                             elif str(cp_descriptor.get("port-security-enabled")).lower() == "true":
1205                                 db_interface["port_security"] = 1
1206                         except KeyError:
1207                             raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'vdu[{vdu}]':"
1208                                                 "'interface[{iface}]':'vnfd-connection-point-ref':'{cp}' is not present"
1209                                                 " at connection-point".format(
1210                                                     vnf=vnfd_id, vdu=vdu_id, iface=iface["name"],
1211                                                     cp=iface.get("vnfd-connection-point-ref")),
1212                                                 httperrors.Bad_Request)
1213                     elif iface.get("internal-connection-point-ref"):
1214                         try:
1215                             for icp_descriptor in vdu_descriptor["internal-connection-point"]:
1216                                 if icp_descriptor["id"] == str(iface.get("internal-connection-point-ref")):
1217                                     break
1218                             else:
1219                                 raise KeyError("does not exist at vdu:internal-connection-point")
1220                             icp = None
1221                             icp_vld = None
1222                             for vld in vnfd.get("internal-vld").values():
1223                                 for cp in vld.get("internal-connection-point").values():
1224                                     if cp.get("id-ref") == iface.get("internal-connection-point-ref"):
1225                                         if icp:
1226                                             raise KeyError("is referenced by more than one 'internal-vld'")
1227                                         icp = cp
1228                                         icp_vld = vld
1229                             if not icp:
1230                                 raise KeyError("is not referenced by any 'internal-vld'")
1231
1232                             # set network type as data
1233                             if iface.get("virtual-interface") and iface["virtual-interface"].get("type") in \
1234                                     ("SR-IOV", "PCI-PASSTHROUGH"):
1235                                 db_nets[net_id2index[icp_vld.get("id")]]["type"] = "data"
1236                             db_interface["net_id"] = net_id2uuid[icp_vld.get("id")]
1237                             if str(icp_descriptor.get("port-security-enabled")).lower() == "false":
1238                                 db_interface["port_security"] = 0
1239                             elif str(icp_descriptor.get("port-security-enabled")).lower() == "true":
1240                                 db_interface["port_security"] = 1
1241                             if icp.get("ip-address"):
1242                                 if not icp_vld.get("ip-profile-ref"):
1243                                     raise NfvoException
1244                                 db_interface["ip_address"] = str(icp.get("ip-address"))
1245                         except KeyError as e:
1246                             raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'vdu[{vdu}]':"
1247                                                 "'interface[{iface}]':'internal-connection-point-ref':'{cp}'"
1248                                                 " {msg}".format(
1249                                                     vnf=vnfd_id, vdu=vdu_id, iface=iface["name"],
1250                                                     cp=iface.get("internal-connection-point-ref"), msg=str(e)),
1251                                                 httperrors.Bad_Request)
1252                     if iface.get("position"):
1253                         db_interface["created_at"] = int(iface.get("position")) * 50
1254                     if iface.get("mac-address"):
1255                         db_interface["mac"] = str(iface.get("mac-address"))
1256                     db_interfaces.append(db_interface)
1257
1258                 # table flavors
1259                 db_flavor = {
1260                     "name": get_str(vdu, "name", 250) + "-flv",
1261                     "vcpus": int(vdu["vm-flavor"].get("vcpu-count", 1)),
1262                     "ram": int(vdu["vm-flavor"].get("memory-mb", 1)),
1263                     "disk": int(vdu["vm-flavor"].get("storage-gb", 0)),
1264                 }
1265                 # TODO revise the case of several numa-node-policy node
1266                 extended = {}
1267                 numa = {}
1268                 if devices:
1269                     extended["devices"] = devices
1270                 if flavor_epa_interfaces:
1271                     numa["interfaces"] = flavor_epa_interfaces
1272                 if vdu.get("guest-epa"):   # TODO or dedicated_int:
1273                     epa_vcpu_set = False
1274                     if vdu["guest-epa"].get("numa-node-policy"):  # TODO or dedicated_int:
1275                         numa_node_policy = vdu["guest-epa"].get("numa-node-policy")
1276                         if numa_node_policy.get("node"):
1277                             numa_node = next(iter(numa_node_policy["node"].values()))
1278                             if numa_node.get("num-cores"):
1279                                 numa["cores"] = numa_node["num-cores"]
1280                                 epa_vcpu_set = True
1281                             if numa_node.get("paired-threads"):
1282                                 if numa_node["paired-threads"].get("num-paired-threads"):
1283                                     numa["paired-threads"] = int(numa_node["paired-threads"]["num-paired-threads"])
1284                                     epa_vcpu_set = True
1285                                 if len(numa_node["paired-threads"].get("paired-thread-ids")):
1286                                     numa["paired-threads-id"] = []
1287                                     for pair in numa_node["paired-threads"]["paired-thread-ids"].values():
1288                                         numa["paired-threads-id"].append(
1289                                             (str(pair["thread-a"]), str(pair["thread-b"]))
1290                                         )
1291                             if numa_node.get("num-threads"):
1292                                 numa["threads"] = int(numa_node["num-threads"])
1293                                 epa_vcpu_set = True
1294                             if numa_node.get("memory-mb"):
1295                                 numa["memory"] = max(int(numa_node["memory-mb"] / 1024), 1)
1296                     if vdu["guest-epa"].get("mempage-size"):
1297                         if vdu["guest-epa"]["mempage-size"] != "SMALL":
1298                             numa["memory"] = max(int(db_flavor["ram"] / 1024), 1)
1299                     if vdu["guest-epa"].get("cpu-pinning-policy") and not epa_vcpu_set:
1300                         if vdu["guest-epa"]["cpu-pinning-policy"] == "DEDICATED":
1301                             if vdu["guest-epa"].get("cpu-thread-pinning-policy") and \
1302                                             vdu["guest-epa"]["cpu-thread-pinning-policy"] != "PREFER":
1303                                 numa["cores"] = max(db_flavor["vcpus"], 1)
1304                             else:
1305                                 numa["threads"] = max(db_flavor["vcpus"], 1)
1306                             epa_vcpu_set = True
1307                     if vdu["guest-epa"].get("cpu-quota") and not epa_vcpu_set:
1308                         cpuquota = get_resource_allocation_params(vdu["guest-epa"].get("cpu-quota"))
1309                         if cpuquota:
1310                             extended["cpu-quota"] = cpuquota
1311                     if vdu["guest-epa"].get("mem-quota"):
1312                         vduquota = get_resource_allocation_params(vdu["guest-epa"].get("mem-quota"))
1313                         if vduquota:
1314                             extended["mem-quota"] = vduquota
1315                     if vdu["guest-epa"].get("disk-io-quota"):
1316                         diskioquota = get_resource_allocation_params(vdu["guest-epa"].get("disk-io-quota"))
1317                         if diskioquota:
1318                             extended["disk-io-quota"] = diskioquota
1319                     if vdu["guest-epa"].get("vif-quota"):
1320                         vifquota = get_resource_allocation_params(vdu["guest-epa"].get("vif-quota"))
1321                         if vifquota:
1322                             extended["vif-quota"] = vifquota
1323                 if numa:
1324                     extended["numas"] = [numa]
1325                 if extended:
1326                     extended_text = yaml.safe_dump(extended, default_flow_style=True, width=256)
1327                     db_flavor["extended"] = extended_text
1328                 # look if flavor exist
1329                 temp_flavor_dict = {'disk': db_flavor.get('disk', 0),
1330                                     'ram': db_flavor.get('ram'),
1331                                     'vcpus': db_flavor.get('vcpus'),
1332                                     'extended': db_flavor.get('extended')
1333                                     }
1334                 existing_flavors = mydb.get_rows(FROM="flavors", WHERE=temp_flavor_dict)
1335                 if existing_flavors:
1336                     flavor_uuid = existing_flavors[0]["uuid"]
1337                 else:
1338                     flavor_uuid = str(uuid4())
1339                     uuid_list.append(flavor_uuid)
1340                     db_flavor["uuid"] = flavor_uuid
1341                     db_flavors.append(db_flavor)
1342                 db_vm["flavor_id"] = flavor_uuid
1343
1344             # VNF affinity and antiaffinity
1345             for pg in vnfd.get("placement-groups").values():
1346                 pg_name = get_str(pg, "name", 255)
1347                 for vdu in pg.get("member-vdus").values():
1348                     vdu_id = get_str(vdu, "member-vdu-ref", 255)
1349                     if vdu_id not in vdu_id2db_table_index:
1350                         raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'placement-groups[{pg}]':"
1351                                             "'member-vdus':'{vdu}'. Reference to a non-existing vdu".format(
1352                                                 vnf=vnfd_id, pg=pg_name, vdu=vdu_id),
1353                                             httperrors.Bad_Request)
1354                     db_vms[vdu_id2db_table_index[vdu_id]]["availability_zone"] = pg_name
1355                     # TODO consider the case of isolation and not colocation
1356                     # if pg.get("strategy") == "ISOLATION":
1357
1358             # VNF mgmt configuration
1359             if vnfd["mgmt-interface"].get("vdu-id"):
1360                 mgmt_vdu_id = get_str(vnfd["mgmt-interface"], "vdu-id", 255)
1361                 if mgmt_vdu_id not in vdu_id2uuid:
1362                     raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'mgmt-interface':'vdu-id':"
1363                                         "'{vdu}'. Reference to a non-existing vdu".format(
1364                                             vnf=vnfd_id, vdu=mgmt_vdu_id),
1365                                         httperrors.Bad_Request)
1366                 mgmt_access["vm_id"] = vdu_id2uuid[mgmt_vdu_id]
1367                 mgmt_access["vdu-id"] = mgmt_vdu_id
1368                 # if only one cp is defined by this VDU, mark this interface as of type "mgmt"
1369                 if vdu_id2cp_name.get(mgmt_vdu_id):
1370                     if cp_name2db_interface[vdu_id2cp_name[mgmt_vdu_id]]:
1371                         cp_name2db_interface[vdu_id2cp_name[mgmt_vdu_id]]["type"] = "mgmt"
1372
1373             if vnfd["mgmt-interface"].get("ip-address"):
1374                 mgmt_access["ip-address"] = str(vnfd["mgmt-interface"].get("ip-address"))
1375             if vnfd["mgmt-interface"].get("cp") and vnfd.get("vdu"):
1376                 if vnfd["mgmt-interface"]["cp"] not in cp_name2iface_uuid:
1377                     raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'mgmt-interface':'cp'['{cp}']. "
1378                                         "Reference to a non-existing connection-point".format(
1379                                             vnf=vnfd_id, cp=vnfd["mgmt-interface"]["cp"]),
1380                                         httperrors.Bad_Request)
1381                 mgmt_access["vm_id"] = cp_name2vm_uuid[vnfd["mgmt-interface"]["cp"]]
1382                 mgmt_access["interface_id"] = cp_name2iface_uuid[vnfd["mgmt-interface"]["cp"]]
1383                 mgmt_access["vdu-id"] = cp_name2vdu_id[vnfd["mgmt-interface"]["cp"]]
1384                 # mark this interface as of type mgmt
1385                 if cp_name2db_interface[vnfd["mgmt-interface"]["cp"]]:
1386                     cp_name2db_interface[vnfd["mgmt-interface"]["cp"]]["type"] = "mgmt"
1387
1388             default_user = get_str(vnfd.get("vnf-configuration", {}).get("config-access", {}).get("ssh-access", {}),
1389                                     "default-user", 64)
1390             if default_user:
1391                 mgmt_access["default_user"] = default_user
1392
1393             required = get_str(vnfd.get("vnf-configuration", {}).get("config-access", {}).get("ssh-access", {}),
1394                                    "required", 6)
1395             if required:
1396                 mgmt_access["required"] = required
1397
1398             password_ = get_str(vnfd.get("vnf-configuration", {}).get("config-access", {}),
1399                                    "password", 64)
1400             if password_:
1401                 mgmt_access["password"] = password_
1402
1403             if mgmt_access:
1404                 db_vnf["mgmt_access"] = yaml.safe_dump(mgmt_access, default_flow_style=True, width=256)
1405
1406             db_vnfs.append(db_vnf)
1407         db_tables=[
1408             {"vnfs": db_vnfs},
1409             {"nets": db_nets},
1410             {"images": db_images},
1411             {"flavors": db_flavors},
1412             {"ip_profiles": db_ip_profiles},
1413             {"vms": db_vms},
1414             {"interfaces": db_interfaces},
1415         ]
1416
1417         logger.debug("create_vnf Deployment done vnfDict: %s",
1418                     yaml.safe_dump(db_tables, indent=4, default_flow_style=False) )
1419         mydb.new_rows(db_tables, uuid_list)
1420         return vnfd_uuid_list
1421     except NfvoException:
1422         raise
1423     except Exception as e:
1424         logger.error("Exception {}".format(e))
1425         raise  # NfvoException("Exception {}".format(e), httperrors.Bad_Request)
1426
1427
1428 @deprecated("Use new_vnfd_v3")
1429 def new_vnf(mydb, tenant_id, vnf_descriptor):
1430     global global_config
1431
1432     # Step 1. Check the VNF descriptor
1433     check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1)
1434     # Step 2. Check tenant exist
1435     vims = {}
1436     if tenant_id != "any":
1437         check_tenant(mydb, tenant_id)
1438         if "tenant_id" in vnf_descriptor["vnf"]:
1439             if vnf_descriptor["vnf"]["tenant_id"] != tenant_id:
1440                 raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(vnf_descriptor["vnf"]["tenant_id"], tenant_id),
1441                                     httperrors.Unauthorized)
1442         else:
1443             vnf_descriptor['vnf']['tenant_id'] = tenant_id
1444         # Step 3. Get the URL of the VIM from the nfvo_tenant and the datacenter
1445         if global_config["auto_push_VNF_to_VIMs"]:
1446             vims = get_vim(mydb, tenant_id, ignore_errors=True)
1447
1448     # Step 4. Review the descriptor and add missing  fields
1449     #print vnf_descriptor
1450     #logger.debug("Refactoring VNF descriptor with fields: description, public (default: true)")
1451     vnf_name = vnf_descriptor['vnf']['name']
1452     vnf_descriptor['vnf']['description'] = vnf_descriptor['vnf'].get("description", vnf_name)
1453     if "physical" in vnf_descriptor['vnf']:
1454         del vnf_descriptor['vnf']['physical']
1455     #print vnf_descriptor
1456
1457     # Step 6. For each VNFC in the descriptor, flavors and images are created in the VIM
1458     logger.debug('BEGIN creation of VNF "%s"' % vnf_name)
1459     logger.debug("VNF %s: consisting of %d VNFC(s)" % (vnf_name,len(vnf_descriptor['vnf']['VNFC'])))
1460
1461     #For each VNFC, we add it to the VNFCDict and we  create a flavor.
1462     VNFCDict = {}     # Dictionary, key: VNFC name, value: dict with the relevant information to create the VNF and VMs in the MANO database
1463     rollback_list = []    # It will contain the new images created in mano. It is used for rollback
1464     try:
1465         logger.debug("Creating additional disk images and new flavors in the VIM for each VNFC")
1466         for vnfc in vnf_descriptor['vnf']['VNFC']:
1467             VNFCitem={}
1468             VNFCitem["name"] = vnfc['name']
1469             VNFCitem["availability_zone"] = vnfc.get('availability_zone')
1470             VNFCitem["description"] = vnfc.get("description", 'VM {} of the VNF {}'.format(vnfc['name'],vnf_name))
1471
1472             #print "Flavor name: %s. Description: %s" % (VNFCitem["name"]+"-flv", VNFCitem["description"])
1473
1474             myflavorDict = {}
1475             myflavorDict["name"] = vnfc['name']+"-flv"   #Maybe we could rename the flavor by using the field "image name" if exists
1476             myflavorDict["description"] = VNFCitem["description"]
1477             myflavorDict["ram"] = vnfc.get("ram", 0)
1478             myflavorDict["vcpus"] = vnfc.get("vcpus", 0)
1479             myflavorDict["disk"] = vnfc.get("disk", 0)
1480             myflavorDict["extended"] = {}
1481
1482             devices = vnfc.get("devices")
1483             if devices != None:
1484                 myflavorDict["extended"]["devices"] = devices
1485
1486             # TODO:
1487             # Mapping from processor models to rankings should be available somehow in the NFVO. They could be taken from VIM or directly from a new database table
1488             # Another option is that the processor in the VNF descriptor specifies directly the ranking of the host
1489
1490             # Previous code has been commented
1491             #if vnfc['processor']['model'] == "Intel(R) Xeon(R) CPU E5-4620 0 @ 2.20GHz" :
1492             #    myflavorDict["flavor"]['extended']['processor_ranking'] = 200
1493             #elif vnfc['processor']['model'] == "Intel(R) Xeon(R) CPU E5-2697 v2 @ 2.70GHz" :
1494             #    myflavorDict["flavor"]['extended']['processor_ranking'] = 300
1495             #else:
1496             #    result2, message = rollback(myvim, myvimURL, myvim_tenant, flavorList, imageList)
1497             #    if result2:
1498             #        print "Error creating flavor: unknown processor model. Rollback successful."
1499             #        return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback successful."
1500             #    else:
1501             #        return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback fail: you need to access VIM and delete the following %s" % message
1502             myflavorDict['extended']['processor_ranking'] = 100  #Hardcoded value, while we decide when the mapping is done
1503
1504             if 'numas' in vnfc and len(vnfc['numas'])>0:
1505                 myflavorDict['extended']['numas'] = vnfc['numas']
1506
1507             #print myflavorDict
1508
1509             # Step 6.2 New flavors are created in the VIM
1510             flavor_id = create_or_use_flavor(mydb, vims, myflavorDict, rollback_list)
1511
1512             #print "Flavor id for VNFC %s: %s" % (vnfc['name'],flavor_id)
1513             VNFCitem["flavor_id"] = flavor_id
1514             VNFCDict[vnfc['name']] = VNFCitem
1515
1516         logger.debug("Creating new images in the VIM for each VNFC")
1517         # Step 6.3 New images are created in the VIM
1518         #For each VNFC, we must create the appropriate image.
1519         #This "for" loop might be integrated with the previous one
1520         #In case this integration is made, the VNFCDict might become a VNFClist.
1521         for vnfc in vnf_descriptor['vnf']['VNFC']:
1522             #print "Image name: %s. Description: %s" % (vnfc['name']+"-img", VNFCDict[vnfc['name']]['description'])
1523             image_dict={}
1524             image_dict['name']=vnfc.get('image name',vnf_name+"-"+vnfc['name']+"-img")
1525             image_dict['universal_name']=vnfc.get('image name')
1526             image_dict['description']=vnfc.get('image name', VNFCDict[vnfc['name']]['description'])
1527             image_dict['location']=vnfc.get('VNFC image')
1528             #image_dict['new_location']=vnfc.get('image location')
1529             image_dict['checksum']=vnfc.get('image checksum')
1530             image_metadata_dict = vnfc.get('image metadata', None)
1531             image_metadata_str = None
1532             if image_metadata_dict is not None:
1533                 image_metadata_str = yaml.safe_dump(image_metadata_dict,default_flow_style=True,width=256)
1534             image_dict['metadata']=image_metadata_str
1535             #print "create_or_use_image", mydb, vims, image_dict, rollback_list
1536             image_id = create_or_use_image(mydb, vims, image_dict, rollback_list)
1537             #print "Image id for VNFC %s: %s" % (vnfc['name'],image_id)
1538             VNFCDict[vnfc['name']]["image_id"] = image_id
1539             VNFCDict[vnfc['name']]["image_path"] = vnfc.get('VNFC image')
1540             VNFCDict[vnfc['name']]["count"] = vnfc.get('count', 1)
1541             if vnfc.get("boot-data"):
1542                 VNFCDict[vnfc['name']]["boot_data"] = yaml.safe_dump(vnfc["boot-data"], default_flow_style=True, width=256)
1543
1544
1545         # Step 7. Storing the VNF descriptor in the repository
1546         if "descriptor" not in vnf_descriptor["vnf"]:
1547             vnf_descriptor["vnf"]["descriptor"] = yaml.safe_dump(vnf_descriptor, indent=4, explicit_start=True, default_flow_style=False)
1548
1549         # Step 8. Adding the VNF to the NFVO DB
1550         vnf_id = mydb.new_vnf_as_a_whole(tenant_id,vnf_name,vnf_descriptor,VNFCDict)
1551         return vnf_id
1552     except (db_base_Exception, vimconn.vimconnException, KeyError) as e:
1553         _, message = rollback(mydb, vims, rollback_list)
1554         if isinstance(e, db_base_Exception):
1555             error_text = "Exception at database"
1556         elif isinstance(e, KeyError):
1557             error_text = "KeyError exception "
1558             e.http_code = httperrors.Internal_Server_Error
1559         else:
1560             error_text = "Exception at VIM"
1561         error_text += " {} {}. {}".format(type(e).__name__, str(e), message)
1562         #logger.error("start_scenario %s", error_text)
1563         raise NfvoException(error_text, e.http_code)
1564
1565
1566 @deprecated("Use new_vnfd_v3")
1567 def new_vnf_v02(mydb, tenant_id, vnf_descriptor):
1568     global global_config
1569
1570     # Step 1. Check the VNF descriptor
1571     check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=2)
1572     # Step 2. Check tenant exist
1573     vims = {}
1574     if tenant_id != "any":
1575         check_tenant(mydb, tenant_id)
1576         if "tenant_id" in vnf_descriptor["vnf"]:
1577             if vnf_descriptor["vnf"]["tenant_id"] != tenant_id:
1578                 raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(vnf_descriptor["vnf"]["tenant_id"], tenant_id),
1579                                     httperrors.Unauthorized)
1580         else:
1581             vnf_descriptor['vnf']['tenant_id'] = tenant_id
1582         # Step 3. Get the URL of the VIM from the nfvo_tenant and the datacenter
1583         if global_config["auto_push_VNF_to_VIMs"]:
1584             vims = get_vim(mydb, tenant_id, ignore_errors=True)
1585
1586     # Step 4. Review the descriptor and add missing  fields
1587     #print vnf_descriptor
1588     #logger.debug("Refactoring VNF descriptor with fields: description, public (default: true)")
1589     vnf_name = vnf_descriptor['vnf']['name']
1590     vnf_descriptor['vnf']['description'] = vnf_descriptor['vnf'].get("description", vnf_name)
1591     if "physical" in vnf_descriptor['vnf']:
1592         del vnf_descriptor['vnf']['physical']
1593     #print vnf_descriptor
1594
1595     # Step 6. For each VNFC in the descriptor, flavors and images are created in the VIM
1596     logger.debug('BEGIN creation of VNF "%s"' % vnf_name)
1597     logger.debug("VNF %s: consisting of %d VNFC(s)" % (vnf_name,len(vnf_descriptor['vnf']['VNFC'])))
1598
1599     #For each VNFC, we add it to the VNFCDict and we  create a flavor.
1600     VNFCDict = {}     # Dictionary, key: VNFC name, value: dict with the relevant information to create the VNF and VMs in the MANO database
1601     rollback_list = []    # It will contain the new images created in mano. It is used for rollback
1602     try:
1603         logger.debug("Creating additional disk images and new flavors in the VIM for each VNFC")
1604         for vnfc in vnf_descriptor['vnf']['VNFC']:
1605             VNFCitem={}
1606             VNFCitem["name"] = vnfc['name']
1607             VNFCitem["description"] = vnfc.get("description", 'VM {} of the VNF {}'.format(vnfc['name'],vnf_name))
1608
1609             #print "Flavor name: %s. Description: %s" % (VNFCitem["name"]+"-flv", VNFCitem["description"])
1610
1611             myflavorDict = {}
1612             myflavorDict["name"] = vnfc['name']+"-flv"   #Maybe we could rename the flavor by using the field "image name" if exists
1613             myflavorDict["description"] = VNFCitem["description"]
1614             myflavorDict["ram"] = vnfc.get("ram", 0)
1615             myflavorDict["vcpus"] = vnfc.get("vcpus", 0)
1616             myflavorDict["disk"] = vnfc.get("disk", 0)
1617             myflavorDict["extended"] = {}
1618
1619             devices = vnfc.get("devices")
1620             if devices != None:
1621                 myflavorDict["extended"]["devices"] = devices
1622
1623             # TODO:
1624             # Mapping from processor models to rankings should be available somehow in the NFVO. They could be taken from VIM or directly from a new database table
1625             # Another option is that the processor in the VNF descriptor specifies directly the ranking of the host
1626
1627             # Previous code has been commented
1628             #if vnfc['processor']['model'] == "Intel(R) Xeon(R) CPU E5-4620 0 @ 2.20GHz" :
1629             #    myflavorDict["flavor"]['extended']['processor_ranking'] = 200
1630             #elif vnfc['processor']['model'] == "Intel(R) Xeon(R) CPU E5-2697 v2 @ 2.70GHz" :
1631             #    myflavorDict["flavor"]['extended']['processor_ranking'] = 300
1632             #else:
1633             #    result2, message = rollback(myvim, myvimURL, myvim_tenant, flavorList, imageList)
1634             #    if result2:
1635             #        print "Error creating flavor: unknown processor model. Rollback successful."
1636             #        return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback successful."
1637             #    else:
1638             #        return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback fail: you need to access VIM and delete the following %s" % message
1639             myflavorDict['extended']['processor_ranking'] = 100  #Hardcoded value, while we decide when the mapping is done
1640
1641             if 'numas' in vnfc and len(vnfc['numas'])>0:
1642                 myflavorDict['extended']['numas'] = vnfc['numas']
1643
1644             #print myflavorDict
1645
1646             # Step 6.2 New flavors are created in the VIM
1647             flavor_id = create_or_use_flavor(mydb, vims, myflavorDict, rollback_list)
1648
1649             #print "Flavor id for VNFC %s: %s" % (vnfc['name'],flavor_id)
1650             VNFCitem["flavor_id"] = flavor_id
1651             VNFCDict[vnfc['name']] = VNFCitem
1652
1653         logger.debug("Creating new images in the VIM for each VNFC")
1654         # Step 6.3 New images are created in the VIM
1655         #For each VNFC, we must create the appropriate image.
1656         #This "for" loop might be integrated with the previous one
1657         #In case this integration is made, the VNFCDict might become a VNFClist.
1658         for vnfc in vnf_descriptor['vnf']['VNFC']:
1659             #print "Image name: %s. Description: %s" % (vnfc['name']+"-img", VNFCDict[vnfc['name']]['description'])
1660             image_dict={}
1661             image_dict['name']=vnfc.get('image name',vnf_name+"-"+vnfc['name']+"-img")
1662             image_dict['universal_name']=vnfc.get('image name')
1663             image_dict['description']=vnfc.get('image name', VNFCDict[vnfc['name']]['description'])
1664             image_dict['location']=vnfc.get('VNFC image')
1665             #image_dict['new_location']=vnfc.get('image location')
1666             image_dict['checksum']=vnfc.get('image checksum')
1667             image_metadata_dict = vnfc.get('image metadata', None)
1668             image_metadata_str = None
1669             if image_metadata_dict is not None:
1670                 image_metadata_str = yaml.safe_dump(image_metadata_dict,default_flow_style=True,width=256)
1671             image_dict['metadata']=image_metadata_str
1672             #print "create_or_use_image", mydb, vims, image_dict, rollback_list
1673             image_id = create_or_use_image(mydb, vims, image_dict, rollback_list)
1674             #print "Image id for VNFC %s: %s" % (vnfc['name'],image_id)
1675             VNFCDict[vnfc['name']]["image_id"] = image_id
1676             VNFCDict[vnfc['name']]["image_path"] = vnfc.get('VNFC image')
1677             VNFCDict[vnfc['name']]["count"] = vnfc.get('count', 1)
1678             if vnfc.get("boot-data"):
1679                 VNFCDict[vnfc['name']]["boot_data"] = yaml.safe_dump(vnfc["boot-data"], default_flow_style=True, width=256)
1680
1681         # Step 7. Storing the VNF descriptor in the repository
1682         if "descriptor" not in vnf_descriptor["vnf"]:
1683             vnf_descriptor["vnf"]["descriptor"] = yaml.safe_dump(vnf_descriptor, indent=4, explicit_start=True, default_flow_style=False)
1684
1685         # Step 8. Adding the VNF to the NFVO DB
1686         vnf_id = mydb.new_vnf_as_a_whole2(tenant_id,vnf_name,vnf_descriptor,VNFCDict)
1687         return vnf_id
1688     except (db_base_Exception, vimconn.vimconnException, KeyError) as e:
1689         _, message = rollback(mydb, vims, rollback_list)
1690         if isinstance(e, db_base_Exception):
1691             error_text = "Exception at database"
1692         elif isinstance(e, KeyError):
1693             error_text = "KeyError exception "
1694             e.http_code = httperrors.Internal_Server_Error
1695         else:
1696             error_text = "Exception at VIM"
1697         error_text += " {} {}. {}".format(type(e).__name__, str(e), message)
1698         #logger.error("start_scenario %s", error_text)
1699         raise NfvoException(error_text, e.http_code)
1700
1701
1702 def get_vnf_id(mydb, tenant_id, vnf_id):
1703     #check valid tenant_id
1704     check_tenant(mydb, tenant_id)
1705     #obtain data
1706     where_or = {}
1707     if tenant_id != "any":
1708         where_or["tenant_id"] = tenant_id
1709         where_or["public"] = True
1710     vnf = mydb.get_table_by_uuid_name('vnfs', vnf_id, "VNF", WHERE_OR=where_or, WHERE_AND_OR="AND")
1711
1712     vnf_id = vnf["uuid"]
1713     filter_keys = ('uuid', 'name', 'description', 'public', "tenant_id", "osm_id", "created_at")
1714     filtered_content = dict( (k,v) for k,v in vnf.items() if k in filter_keys )
1715     #change_keys_http2db(filtered_content, http2db_vnf, reverse=True)
1716     data={'vnf' : filtered_content}
1717     #GET VM
1718     content = mydb.get_rows(FROM='vnfs join vms on vnfs.uuid=vms.vnf_id',
1719             SELECT=('vms.uuid as uuid', 'vms.osm_id as osm_id', 'vms.name as name', 'vms.description as description',
1720                     'boot_data'),
1721             WHERE={'vnfs.uuid': vnf_id} )
1722     if len(content) != 0:
1723         #raise NfvoException("vnf '{}' not found".format(vnf_id), httperrors.Not_Found)
1724     # change boot_data into boot-data
1725         for vm in content:
1726             if vm.get("boot_data"):
1727                 vm["boot-data"] = yaml.safe_load(vm["boot_data"])
1728                 del vm["boot_data"]
1729
1730         data['vnf']['VNFC'] = content
1731     #TODO: GET all the information from a VNFC and include it in the output.
1732
1733     #GET NET
1734     content = mydb.get_rows(FROM='vnfs join nets on vnfs.uuid=nets.vnf_id',
1735                                     SELECT=('nets.uuid as uuid','nets.name as name','nets.description as description', 'nets.type as type', 'nets.multipoint as multipoint'),
1736                                     WHERE={'vnfs.uuid': vnf_id} )
1737     data['vnf']['nets'] = content
1738
1739     #GET ip-profile for each net
1740     for net in data['vnf']['nets']:
1741         ipprofiles = mydb.get_rows(FROM='ip_profiles',
1742                                    SELECT=('ip_version','subnet_address','gateway_address','dns_address','dhcp_enabled','dhcp_start_address','dhcp_count'),
1743                                    WHERE={'net_id': net["uuid"]} )
1744         if len(ipprofiles)==1:
1745             net["ip_profile"] = ipprofiles[0]
1746         elif len(ipprofiles)>1:
1747             raise NfvoException("More than one ip-profile found with this criteria: net_id='{}'".format(net['uuid']), httperrors.Bad_Request)
1748
1749
1750     #TODO: For each net, GET its elements and relevant info per element (VNFC, iface, ip_address) and include them in the output.
1751
1752     #GET External Interfaces
1753     content = mydb.get_rows(FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces on vms.uuid=interfaces.vm_id',\
1754                                     SELECT=('interfaces.uuid as uuid','interfaces.external_name as external_name', 'vms.name as vm_name', 'interfaces.vm_id as vm_id', \
1755                                             'interfaces.internal_name as internal_name', 'interfaces.type as type', 'interfaces.vpci as vpci','interfaces.bw as bw'),\
1756                                     WHERE={'vnfs.uuid': vnf_id, 'interfaces.external_name<>': None} )
1757     #print content
1758     data['vnf']['external-connections'] = content
1759
1760     return data
1761
1762
1763 def delete_vnf(mydb,tenant_id,vnf_id,datacenter=None,vim_tenant=None):
1764     # Check tenant exist
1765     if tenant_id != "any":
1766         check_tenant(mydb, tenant_id)
1767         # Get the URL of the VIM from the nfvo_tenant and the datacenter
1768         vims = get_vim(mydb, tenant_id, ignore_errors=True)
1769     else:
1770         vims={}
1771
1772     # Checking if it is a valid uuid and, if not, getting the uuid assuming that the name was provided"
1773     where_or = {}
1774     if tenant_id != "any":
1775         where_or["tenant_id"] = tenant_id
1776         where_or["public"] = True
1777     vnf = mydb.get_table_by_uuid_name('vnfs', vnf_id, "VNF", WHERE_OR=where_or, WHERE_AND_OR="AND")
1778     vnf_id = vnf["uuid"]
1779
1780     # "Getting the list of flavors and tenants of the VNF"
1781     flavorList = get_flavorlist(mydb, vnf_id)
1782     if len(flavorList)==0:
1783         logger.warn("delete_vnf error. No flavors found for the VNF id '%s'", vnf_id)
1784
1785     imageList = get_imagelist(mydb, vnf_id)
1786     if len(imageList)==0:
1787         logger.warn( "delete_vnf error. No images found for the VNF id '%s'", vnf_id)
1788
1789     deleted = mydb.delete_row_by_id('vnfs', vnf_id)
1790     if deleted == 0:
1791         raise NfvoException("vnf '{}' not found".format(vnf_id), httperrors.Not_Found)
1792
1793     undeletedItems = []
1794     for flavor in flavorList:
1795         #check if flavor is used by other vnf
1796         try:
1797             c = mydb.get_rows(FROM='vms', WHERE={'flavor_id':flavor} )
1798             if len(c) > 0:
1799                 logger.debug("Flavor '%s' not deleted because it is being used by another VNF", flavor)
1800                 continue
1801             #flavor not used, must be deleted
1802             #delelte at VIM
1803             c = mydb.get_rows(FROM='datacenters_flavors', WHERE={'flavor_id': flavor})
1804             for flavor_vim in c:
1805                 if not flavor_vim['created']:  # skip this flavor because not created by openmano
1806                     continue
1807                 # look for vim
1808                 myvim = None
1809                 for vim in vims.values():
1810                     if vim["config"]["datacenter_tenant_id"] == flavor_vim["datacenter_vim_id"]:
1811                         myvim = vim
1812                         break
1813                 if not myvim:
1814                     continue
1815                 try:
1816                     myvim.delete_flavor(flavor_vim["vim_id"])
1817                 except vimconn.vimconnNotFoundException:
1818                     logger.warn("VIM flavor %s not exist at datacenter %s", flavor_vim["vim_id"],
1819                                 flavor_vim["datacenter_vim_id"] )
1820                 except vimconn.vimconnException as e:
1821                     logger.error("Not possible to delete VIM flavor %s from datacenter %s: %s %s",
1822                             flavor_vim["vim_id"], flavor_vim["datacenter_vim_id"], type(e).__name__, str(e))
1823                     undeletedItems.append("flavor {} from VIM {}".format(flavor_vim["vim_id"],
1824                                                                          flavor_vim["datacenter_vim_id"]))
1825             # delete flavor from Database, using table flavors and with cascade foreign key also at datacenters_flavors
1826             mydb.delete_row_by_id('flavors', flavor)
1827         except db_base_Exception as e:
1828             logger.error("delete_vnf_error. Not possible to get flavor details and delete '%s'. %s", flavor, str(e))
1829             undeletedItems.append("flavor {}".format(flavor))
1830
1831
1832     for image in imageList:
1833         try:
1834             #check if image is used by other vnf
1835             c = mydb.get_rows(FROM='vms', WHERE=[{'image_id': image}, {'image_list LIKE ': '%' + image + '%'}])
1836             if len(c) > 0:
1837                 logger.debug("Image '%s' not deleted because it is being used by another VNF", image)
1838                 continue
1839             #image not used, must be deleted
1840             #delelte at VIM
1841             c = mydb.get_rows(FROM='datacenters_images', WHERE={'image_id':image})
1842             for image_vim in c:
1843                 if image_vim["datacenter_vim_id"] not in vims:   # TODO change to datacenter_tenant_id
1844                     continue
1845                 if image_vim['created']=='false': #skip this image because not created by openmano
1846                     continue
1847                 myvim=vims[ image_vim["datacenter_id"] ]
1848                 try:
1849                     myvim.delete_image(image_vim["vim_id"])
1850                 except vimconn.vimconnNotFoundException as e:
1851                     logger.warn("VIM image %s not exist at datacenter %s", image_vim["vim_id"], image_vim["datacenter_id"] )
1852                 except vimconn.vimconnException as e:
1853                     logger.error("Not possible to delete VIM image %s from datacenter %s: %s %s",
1854                             image_vim["vim_id"], image_vim["datacenter_id"], type(e).__name__, str(e))
1855                     undeletedItems.append("image {} from VIM {}".format(image_vim["vim_id"], image_vim["datacenter_id"] ))
1856             #delete image from Database, using table images and with cascade foreign key also at datacenters_images
1857             mydb.delete_row_by_id('images', image)
1858         except db_base_Exception as e:
1859             logger.error("delete_vnf_error. Not possible to get image details and delete '%s'. %s", image, str(e))
1860             undeletedItems.append("image {}".format(image))
1861
1862     return vnf_id + " " + vnf["name"]
1863     #if undeletedItems:
1864     #    return "delete_vnf. Undeleted: %s" %(undeletedItems)
1865
1866
1867 @deprecated("Not used")
1868 def get_hosts_info(mydb, nfvo_tenant_id, datacenter_name=None):
1869     result, vims = get_vim(mydb, nfvo_tenant_id, None, datacenter_name)
1870     if result < 0:
1871         return result, vims
1872     elif result == 0:
1873         return -httperrors.Not_Found, "datacenter '{}' not found".format(datacenter_name)
1874     myvim = next(iter(vims.values()))
1875     result,servers =  myvim.get_hosts_info()
1876     if result < 0:
1877         return result, servers
1878     topology = {'name':myvim['name'] , 'servers': servers}
1879     return result, topology
1880
1881
1882 def get_hosts(mydb, nfvo_tenant_id):
1883     vims = get_vim(mydb, nfvo_tenant_id)
1884     if len(vims) == 0:
1885         raise NfvoException("No datacenter found for tenant '{}'".format(str(nfvo_tenant_id)), httperrors.Not_Found)
1886     elif len(vims)>1:
1887         #print "nfvo.datacenter_action() error. Several datacenters found"
1888         raise NfvoException("More than one datacenters found, try to identify with uuid", httperrors.Conflict)
1889     myvim = next(iter(vims.values()))
1890     try:
1891         hosts =  myvim.get_hosts()
1892         logger.debug('VIM hosts response: '+ yaml.safe_dump(hosts, indent=4, default_flow_style=False))
1893
1894         datacenter = {'Datacenters': [ {'name':myvim['name'],'servers':[]} ] }
1895         for host in hosts:
1896             server={'name':host['name'], 'vms':[]}
1897             for vm in host['instances']:
1898                 #get internal name and model
1899                 try:
1900                     c = mydb.get_rows(SELECT=('name',), FROM='instance_vms as iv join vms on iv.vm_id=vms.uuid',\
1901                         WHERE={'vim_vm_id':vm['id']} )
1902                     if len(c) == 0:
1903                         logger.warn("nfvo.get_hosts virtual machine at VIM '{}' not found at tidnfvo".format(vm['id']))
1904                         continue
1905                     server['vms'].append( {'name':vm['name'] , 'model':c[0]['name']} )
1906
1907                 except db_base_Exception as e:
1908                     logger.warn("nfvo.get_hosts virtual machine at VIM '{}' error {}".format(vm['id'], str(e)))
1909             datacenter['Datacenters'][0]['servers'].append(server)
1910         #return -400, "en construccion"
1911
1912         #print 'datacenters '+ json.dumps(datacenter, indent=4)
1913         return datacenter
1914     except vimconn.vimconnException as e:
1915         raise NfvoException("Not possible to get_host_list from VIM: {}".format(str(e)), e.http_code)
1916
1917
1918 @deprecated("Use new_nsd_v3")
1919 def new_scenario(mydb, tenant_id, topo):
1920
1921 #    result, vims = get_vim(mydb, tenant_id)
1922 #    if result < 0:
1923 #        return result, vims
1924 #1: parse input
1925     if tenant_id != "any":
1926         check_tenant(mydb, tenant_id)
1927         if "tenant_id" in topo:
1928             if topo["tenant_id"] != tenant_id:
1929                 raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(topo["tenant_id"], tenant_id),
1930                                     httperrors.Unauthorized)
1931     else:
1932         tenant_id=None
1933
1934 #1.1: get VNFs and external_networks (other_nets).
1935     vnfs={}
1936     other_nets={}  #external_networks, bridge_networks and data_networkds
1937     nodes = topo['topology']['nodes']
1938     for k in nodes.keys():
1939         if nodes[k]['type'] == 'VNF':
1940             vnfs[k] = nodes[k]
1941             vnfs[k]['ifaces'] = {}
1942         elif nodes[k]['type'] == 'other_network' or nodes[k]['type'] == 'external_network':
1943             other_nets[k] = nodes[k]
1944             other_nets[k]['external']=True
1945         elif nodes[k]['type'] == 'network':
1946             other_nets[k] = nodes[k]
1947             other_nets[k]['external']=False
1948
1949
1950 #1.2: Check that VNF are present at database table vnfs. Insert uuid, description and external interfaces
1951     for name,vnf in vnfs.items():
1952         where = {"OR": {"tenant_id": tenant_id, 'public': "true"}}
1953         error_text = ""
1954         error_pos = "'topology':'nodes':'" + name + "'"
1955         if 'vnf_id' in vnf:
1956             error_text += " 'vnf_id' " +  vnf['vnf_id']
1957             where['uuid'] = vnf['vnf_id']
1958         if 'VNF model' in vnf:
1959             error_text += " 'VNF model' " +  vnf['VNF model']
1960             where['name'] = vnf['VNF model']
1961         if len(where) == 1:
1962             raise NfvoException("Descriptor need a 'vnf_id' or 'VNF model' field at " + error_pos, httperrors.Bad_Request)
1963
1964         vnf_db = mydb.get_rows(SELECT=('uuid','name','description'),
1965                                FROM='vnfs',
1966                                WHERE=where)
1967         if len(vnf_db)==0:
1968             raise NfvoException("unknown" + error_text + " at " + error_pos, httperrors.Not_Found)
1969         elif len(vnf_db)>1:
1970             raise NfvoException("more than one" + error_text + " at " + error_pos + " Concrete with 'vnf_id'", httperrors.Conflict)
1971         vnf['uuid']=vnf_db[0]['uuid']
1972         vnf['description']=vnf_db[0]['description']
1973         #get external interfaces
1974         ext_ifaces = mydb.get_rows(SELECT=('external_name as name','i.uuid as iface_uuid', 'i.type as type'),
1975             FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces as i on vms.uuid=i.vm_id',
1976             WHERE={'vnfs.uuid':vnf['uuid'], 'external_name<>': None} )
1977         for ext_iface in ext_ifaces:
1978             vnf['ifaces'][ ext_iface['name'] ] = {'uuid':ext_iface['iface_uuid'], 'type':ext_iface['type']}
1979
1980 #1.4 get list of connections
1981     conections = topo['topology']['connections']
1982     conections_list = []
1983     conections_list_name = []
1984     for k in conections.keys():
1985         if type(conections[k]['nodes'])==dict: #dict with node:iface pairs
1986             ifaces_list = conections[k]['nodes'].items()
1987         elif type(conections[k]['nodes'])==list: #list with dictionary
1988             ifaces_list=[]
1989             conection_pair_list = map(lambda x: x.items(), conections[k]['nodes'] )
1990             for k2 in conection_pair_list:
1991                 ifaces_list += k2
1992
1993         con_type = conections[k].get("type", "link")
1994         if con_type != "link":
1995             if k in other_nets:
1996                 raise NfvoException("Format error. Reapeted network name at 'topology':'connections':'{}'".format(str(k)), httperrors.Bad_Request)
1997             other_nets[k] = {'external': False}
1998             if conections[k].get("graph"):
1999                 other_nets[k]["graph"] =   conections[k]["graph"]
2000             ifaces_list.append( (k, None) )
2001
2002
2003         if con_type == "external_network":
2004             other_nets[k]['external'] = True
2005             if conections[k].get("model"):
2006                 other_nets[k]["model"] =   conections[k]["model"]
2007             else:
2008                 other_nets[k]["model"] =   k
2009         if con_type == "dataplane_net" or con_type == "bridge_net":
2010             other_nets[k]["model"] = con_type
2011
2012         conections_list_name.append(k)
2013         conections_list.append(set(ifaces_list)) #from list to set to operate as a set (this conversion removes elements that are repeated in a list)
2014         #print set(ifaces_list)
2015     #check valid VNF and iface names
2016         for iface in ifaces_list:
2017             if iface[0] not in vnfs and iface[0] not in other_nets :
2018                 raise NfvoException("format error. Invalid VNF name at 'topology':'connections':'{}':'nodes':'{}'".format(
2019                                                                                         str(k), iface[0]), httperrors.Not_Found)
2020             if iface[0] in vnfs and iface[1] not in vnfs[ iface[0] ]['ifaces']:
2021                 raise NfvoException("format error. Invalid interface name at 'topology':'connections':'{}':'nodes':'{}':'{}'".format(
2022                                                                                         str(k), iface[0], iface[1]), httperrors.Not_Found)
2023
2024 #1.5 unify connections from the pair list to a consolidated list
2025     index=0
2026     while index < len(conections_list):
2027         index2 = index+1
2028         while index2 < len(conections_list):
2029             if len(conections_list[index] & conections_list[index2])>0: #common interface, join nets
2030                 conections_list[index] |= conections_list[index2]
2031                 del conections_list[index2]
2032                 del conections_list_name[index2]
2033             else:
2034                 index2 += 1
2035         conections_list[index] = list(conections_list[index])  # from set to list again
2036         index += 1
2037     #for k in conections_list:
2038     #    print k
2039
2040
2041
2042 #1.6 Delete non external nets
2043 #    for k in other_nets.keys():
2044 #        if other_nets[k]['model']=='bridge' or other_nets[k]['model']=='dataplane_net' or other_nets[k]['model']=='bridge_net':
2045 #            for con in conections_list:
2046 #                delete_indexes=[]
2047 #                for index in range(0,len(con)):
2048 #                    if con[index][0] == k: delete_indexes.insert(0,index) #order from higher to lower
2049 #                for index in delete_indexes:
2050 #                    del con[index]
2051 #            del other_nets[k]
2052 #1.7: Check external_ports are present at database table datacenter_nets
2053     for k,net in other_nets.items():
2054         error_pos = "'topology':'nodes':'" + k + "'"
2055         if net['external']==False:
2056             if 'name' not in net:
2057                 net['name']=k
2058             if 'model' not in net:
2059                 raise NfvoException("needed a 'model' at " + error_pos, httperrors.Bad_Request)
2060             if net['model']=='bridge_net':
2061                 net['type']='bridge';
2062             elif net['model']=='dataplane_net':
2063                 net['type']='data';
2064             else:
2065                 raise NfvoException("unknown 'model' '"+ net['model'] +"' at " + error_pos, httperrors.Not_Found)
2066         else: #external
2067 #IF we do not want to check that external network exist at datacenter
2068             pass
2069 #ELSE
2070 #             error_text = ""
2071 #             WHERE_={}
2072 #             if 'net_id' in net:
2073 #                 error_text += " 'net_id' " +  net['net_id']
2074 #                 WHERE_['uuid'] = net['net_id']
2075 #             if 'model' in net:
2076 #                 error_text += " 'model' " +  net['model']
2077 #                 WHERE_['name'] = net['model']
2078 #             if len(WHERE_) == 0:
2079 #                 return -httperrors.Bad_Request, "needed a 'net_id' or 'model' at " + error_pos
2080 #             r,net_db = mydb.get_table(SELECT=('uuid','name','description','type','shared'),
2081 #                 FROM='datacenter_nets', WHERE=WHERE_ )
2082 #             if r<0:
2083 #                 print "nfvo.new_scenario Error getting datacenter_nets",r,net_db
2084 #             elif r==0:
2085 #                 print "nfvo.new_scenario Error" +error_text+ " is not present at database"
2086 #                 return -httperrors.Bad_Request, "unknown " +error_text+ " at " + error_pos
2087 #             elif r>1:
2088 #                 print "nfvo.new_scenario Error more than one external_network for " +error_text+ " is present at database"
2089 #                 return -httperrors.Bad_Request, "more than one external_network for " +error_text+ "at "+ error_pos + " Concrete with 'net_id'"
2090 #             other_nets[k].update(net_db[0])
2091 #ENDIF
2092     net_list={}
2093     net_nb=0  #Number of nets
2094     for con in conections_list:
2095         #check if this is connected to a external net
2096         other_net_index=-1
2097         #print
2098         #print "con", con
2099         for index in range(0,len(con)):
2100             #check if this is connected to a external net
2101             for net_key in other_nets.keys():
2102                 if con[index][0]==net_key:
2103                     if other_net_index>=0:
2104                         error_text = "There is some interface connected both to net '{}' and net '{}'".format(
2105                             con[other_net_index][0], net_key)
2106                         #print "nfvo.new_scenario " + error_text
2107                         raise NfvoException(error_text, httperrors.Bad_Request)
2108                     else:
2109                         other_net_index = index
2110                         net_target = net_key
2111                     break
2112         #print "other_net_index", other_net_index
2113         try:
2114             if other_net_index>=0:
2115                 del con[other_net_index]
2116 #IF we do not want to check that external network exist at datacenter
2117                 if other_nets[net_target]['external'] :
2118                     if "name" not in other_nets[net_target]:
2119                         other_nets[net_target]['name'] =  other_nets[net_target]['model']
2120                     if other_nets[net_target]["type"] == "external_network":
2121                         if vnfs[ con[0][0] ]['ifaces'][ con[0][1] ]["type"] == "data":
2122                             other_nets[net_target]["type"] =  "data"
2123                         else:
2124                             other_nets[net_target]["type"] =  "bridge"
2125 #ELSE
2126 #                 if other_nets[net_target]['external'] :
2127 #                     type_='data' if len(con)>1 else 'ptp'  #an external net is connected to a external port, so it is ptp if only one connection is done to this net
2128 #                     if type_=='data' and other_nets[net_target]['type']=="ptp":
2129 #                         error_text = "Error connecting %d nodes on a not multipoint net %s" % (len(con), net_target)
2130 #                         print "nfvo.new_scenario " + error_text
2131 #                         return -httperrors.Bad_Request, error_text
2132 #ENDIF
2133                 for iface in con:
2134                     vnfs[ iface[0] ]['ifaces'][ iface[1] ]['net_key'] = net_target
2135             else:
2136                 #create a net
2137                 net_type_bridge=False
2138                 net_type_data=False
2139                 net_target = "__-__net"+str(net_nb)
2140                 net_list[net_target] = {'name': conections_list_name[net_nb],  #"net-"+str(net_nb),
2141                     'description':"net-{} in scenario {}".format(net_nb,topo['name']),
2142                     'external':False}
2143                 for iface in con:
2144                     vnfs[ iface[0] ]['ifaces'][ iface[1] ]['net_key'] = net_target
2145                     iface_type = vnfs[ iface[0] ]['ifaces'][ iface[1] ]['type']
2146                     if iface_type=='mgmt' or iface_type=='bridge':
2147                         net_type_bridge = True
2148                     else:
2149                         net_type_data = True
2150                 if net_type_bridge and net_type_data:
2151                     error_text = "Error connection interfaces of bridge type with data type. Firs node {}, iface {}".format(iface[0], iface[1])
2152                     #print "nfvo.new_scenario " + error_text
2153                     raise NfvoException(error_text, httperrors.Bad_Request)
2154                 elif net_type_bridge:
2155                     type_='bridge'
2156                 else:
2157                     type_='data' if len(con)>2 else 'ptp'
2158                 net_list[net_target]['type'] = type_
2159                 net_nb+=1
2160         except Exception:
2161             error_text = "Error connection node {} : {} does not match any VNF or interface".format(iface[0], iface[1])
2162             #print "nfvo.new_scenario " + error_text
2163             #raise e
2164             raise NfvoException(error_text, httperrors.Bad_Request)
2165
2166 #1.8: Connect to management net all not already connected interfaces of type 'mgmt'
2167     #1.8.1 obtain management net
2168     mgmt_net = mydb.get_rows(SELECT=('uuid','name','description','type','shared'),
2169         FROM='datacenter_nets', WHERE={'name':'mgmt'} )
2170     #1.8.2 check all interfaces from all vnfs
2171     if len(mgmt_net)>0:
2172         add_mgmt_net = False
2173         for vnf in vnfs.values():
2174             for iface in vnf['ifaces'].values():
2175                 if iface['type']=='mgmt' and 'net_key' not in iface:
2176                     #iface not connected
2177                     iface['net_key'] = 'mgmt'
2178                     add_mgmt_net = True
2179         if add_mgmt_net and 'mgmt' not in net_list:
2180             net_list['mgmt']=mgmt_net[0]
2181             net_list['mgmt']['external']=True
2182             net_list['mgmt']['graph']={'visible':False}
2183
2184     net_list.update(other_nets)
2185     #print
2186     #print 'net_list', net_list
2187     #print
2188     #print 'vnfs', vnfs
2189     #print
2190
2191 #2: insert scenario. filling tables scenarios,sce_vnfs,sce_interfaces,sce_nets
2192     c = mydb.new_scenario( { 'vnfs':vnfs, 'nets':net_list,
2193         'tenant_id':tenant_id, 'name':topo['name'],
2194          'description':topo.get('description',topo['name']),
2195          'public': topo.get('public', False)
2196          })
2197
2198     return c
2199
2200
2201 @deprecated("Use new_nsd_v3")
2202 def new_scenario_v02(mydb, tenant_id, scenario_dict, version):
2203     """ This creates a new scenario for version 0.2 and 0.3"""
2204     scenario = scenario_dict["scenario"]
2205     if tenant_id != "any":
2206         check_tenant(mydb, tenant_id)
2207         if "tenant_id" in scenario:
2208             if scenario["tenant_id"] != tenant_id:
2209                 # print "nfvo.new_scenario_v02() tenant '%s' not found" % tenant_id
2210                 raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(
2211                                                     scenario["tenant_id"], tenant_id), httperrors.Unauthorized)
2212     else:
2213         tenant_id=None
2214
2215     # 1: Check that VNF are present at database table vnfs and update content into scenario dict
2216     for name,vnf in scenario["vnfs"].items():
2217         where = {"OR": {"tenant_id": tenant_id, 'public': "true"}}
2218         error_text = ""
2219         error_pos = "'scenario':'vnfs':'" + name + "'"
2220         if 'vnf_id' in vnf:
2221             error_text += " 'vnf_id' " + vnf['vnf_id']
2222             where['uuid'] = vnf['vnf_id']
2223         if 'vnf_name' in vnf:
2224             error_text += " 'vnf_name' " + vnf['vnf_name']
2225             where['name'] = vnf['vnf_name']
2226         if len(where) == 1:
2227             raise NfvoException("Needed a 'vnf_id' or 'vnf_name' at " + error_pos, httperrors.Bad_Request)
2228         vnf_db = mydb.get_rows(SELECT=('uuid', 'name', 'description'),
2229                                FROM='vnfs',
2230                                WHERE=where)
2231         if len(vnf_db) == 0:
2232             raise NfvoException("Unknown" + error_text + " at " + error_pos, httperrors.Not_Found)
2233         elif len(vnf_db) > 1:
2234             raise NfvoException("More than one" + error_text + " at " + error_pos + " Concrete with 'vnf_id'", httperrors.Conflict)
2235         vnf['uuid'] = vnf_db[0]['uuid']
2236         vnf['description'] = vnf_db[0]['description']
2237         vnf['ifaces'] = {}
2238         # get external interfaces
2239         ext_ifaces = mydb.get_rows(SELECT=('external_name as name', 'i.uuid as iface_uuid', 'i.type as type'),
2240                                    FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces as i on vms.uuid=i.vm_id',
2241                                    WHERE={'vnfs.uuid':vnf['uuid'], 'external_name<>': None} )
2242         for ext_iface in ext_ifaces:
2243             vnf['ifaces'][ ext_iface['name'] ] = {'uuid':ext_iface['iface_uuid'], 'type': ext_iface['type']}
2244         # TODO? get internal-connections from db.nets and their profiles, and update scenario[vnfs][internal-connections] accordingly
2245
2246     # 2: Insert net_key and ip_address at every vnf interface
2247     for net_name, net in scenario["networks"].items():
2248         net_type_bridge = False
2249         net_type_data = False
2250         for iface_dict in net["interfaces"]:
2251             if version == "0.2":
2252                 temp_dict = iface_dict
2253                 ip_address = None
2254             elif version == "0.3":
2255                 temp_dict = {iface_dict["vnf"] : iface_dict["vnf_interface"]}
2256                 ip_address = iface_dict.get('ip_address', None)
2257             for vnf, iface in temp_dict.items():
2258                 if vnf not in scenario["vnfs"]:
2259                     error_text = "Error at 'networks':'{}':'interfaces' VNF '{}' not match any VNF at 'vnfs'".format(
2260                         net_name, vnf)
2261                     # logger.debug("nfvo.new_scenario_v02 " + error_text)
2262                     raise NfvoException(error_text, httperrors.Not_Found)
2263                 if iface not in scenario["vnfs"][vnf]['ifaces']:
2264                     error_text = "Error at 'networks':'{}':'interfaces':'{}' interface not match any VNF interface"\
2265                         .format(net_name, iface)
2266                     # logger.debug("nfvo.new_scenario_v02 " + error_text)
2267                     raise NfvoException(error_text, httperrors.Bad_Request)
2268                 if "net_key" in scenario["vnfs"][vnf]['ifaces'][iface]:
2269                     error_text = "Error at 'networks':'{}':'interfaces':'{}' interface already connected at network"\
2270                                  "'{}'".format(net_name, iface,scenario["vnfs"][vnf]['ifaces'][iface]['net_key'])
2271                     # logger.debug("nfvo.new_scenario_v02 " + error_text)
2272                     raise NfvoException(error_text, httperrors.Bad_Request)
2273                 scenario["vnfs"][vnf]['ifaces'][ iface ]['net_key'] = net_name
2274                 scenario["vnfs"][vnf]['ifaces'][iface]['ip_address'] = ip_address
2275                 iface_type = scenario["vnfs"][vnf]['ifaces'][iface]['type']
2276                 if iface_type == 'mgmt' or iface_type == 'bridge':
2277                     net_type_bridge = True
2278                 else:
2279                     net_type_data = True
2280
2281         if net_type_bridge and net_type_data:
2282             error_text = "Error connection interfaces of 'bridge' type and 'data' type at 'networks':'{}':'interfaces'"\
2283                 .format(net_name)
2284             # logger.debug("nfvo.new_scenario " + error_text)
2285             raise NfvoException(error_text, httperrors.Bad_Request)
2286         elif net_type_bridge:
2287             type_ = 'bridge'
2288         else:
2289             type_ = 'data' if len(net["interfaces"]) > 2 else 'ptp'
2290
2291         if net.get("implementation"):     # for v0.3
2292             if type_ == "bridge" and net["implementation"] == "underlay":
2293                 error_text = "Error connecting interfaces of data type to a network declared as 'underlay' at "\
2294                              "'network':'{}'".format(net_name)
2295                 # logger.debug(error_text)
2296                 raise NfvoException(error_text, httperrors.Bad_Request)
2297             elif type_ != "bridge" and net["implementation"] == "overlay":
2298                 error_text = "Error connecting interfaces of data type to a network declared as 'overlay' at "\
2299                              "'network':'{}'".format(net_name)
2300                 # logger.debug(error_text)
2301                 raise NfvoException(error_text, httperrors.Bad_Request)
2302             net.pop("implementation")
2303         if "type" in net and version == "0.3":   # for v0.3
2304             if type_ == "data" and net["type"] == "e-line":
2305                 error_text = "Error connecting more than 2 interfaces of data type to a network declared as type "\
2306                              "'e-line' at 'network':'{}'".format(net_name)
2307                 # logger.debug(error_text)
2308                 raise NfvoException(error_text, httperrors.Bad_Request)
2309             elif type_ == "ptp" and net["type"] == "e-lan":
2310                 type_ = "data"
2311
2312         net['type'] = type_
2313         net['name'] = net_name
2314         net['external'] = net.get('external', False)
2315
2316     # 3: insert at database
2317     scenario["nets"] = scenario["networks"]
2318     scenario['tenant_id'] = tenant_id
2319     scenario_id = mydb.new_scenario(scenario)
2320     return scenario_id
2321
2322
2323 def new_nsd_v3(mydb, tenant_id, nsd_descriptor):
2324     """
2325     Parses an OSM IM nsd_catalog and insert at DB
2326     :param mydb:
2327     :param tenant_id:
2328     :param nsd_descriptor:
2329     :return: The list of created NSD ids
2330     """
2331     try:
2332         mynsd = nsd_catalog.nsd()
2333         try:
2334             pybindJSONDecoder.load_ietf_json(nsd_descriptor, None, None, obj=mynsd, skip_unknown=True)
2335         except Exception as e:
2336             raise NfvoException("Error. Invalid NS descriptor format: " + str(e), httperrors.Bad_Request)
2337         db_scenarios = []
2338         db_sce_nets = []
2339         db_sce_vnfs = []
2340         db_sce_interfaces = []
2341         db_sce_vnffgs = []
2342         db_sce_rsps = []
2343         db_sce_rsp_hops = []
2344         db_sce_classifiers = []
2345         db_sce_classifier_matches = []
2346         db_ip_profiles = []
2347         db_ip_profiles_index = 0
2348         uuid_list = []
2349         nsd_uuid_list = []
2350         for nsd_yang in mynsd.nsd_catalog.nsd.values():
2351             nsd = nsd_yang.get()
2352
2353             # table scenarios
2354             scenario_uuid = str(uuid4())
2355             uuid_list.append(scenario_uuid)
2356             nsd_uuid_list.append(scenario_uuid)
2357             db_scenario = {
2358                 "uuid": scenario_uuid,
2359                 "osm_id": get_str(nsd, "id", 255),
2360                 "name": get_str(nsd, "name", 255),
2361                 "description": get_str(nsd, "description", 255),
2362                 "tenant_id": tenant_id,
2363                 "vendor": get_str(nsd, "vendor", 255),
2364                 "short_name": get_str(nsd, "short-name", 255),
2365                 "descriptor": str(nsd_descriptor)[:60000],
2366             }
2367             db_scenarios.append(db_scenario)
2368
2369             # table sce_vnfs (constituent-vnfd)
2370             vnf_index2scevnf_uuid = {}
2371             vnf_index2vnf_uuid = {}
2372             for vnf in nsd.get("constituent-vnfd").values():
2373                 existing_vnf = mydb.get_rows(FROM="vnfs", WHERE={'osm_id': str(vnf["vnfd-id-ref"])[:255],
2374                                                                       'tenant_id': tenant_id})
2375                 if not existing_vnf:
2376                     raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'constituent-vnfd':'vnfd-id-ref':"
2377                                         "'{}'. Reference to a non-existing VNFD in the catalog".format(
2378                                             str(nsd["id"]), str(vnf["vnfd-id-ref"])[:255]),
2379                                         httperrors.Bad_Request)
2380                 sce_vnf_uuid = str(uuid4())
2381                 uuid_list.append(sce_vnf_uuid)
2382                 db_sce_vnf = {
2383                     "uuid": sce_vnf_uuid,
2384                     "scenario_id": scenario_uuid,
2385                     # "name": get_str(vnf, "member-vnf-index", 255),
2386                     "name": existing_vnf[0]["name"][:200] + "." + get_str(vnf, "member-vnf-index", 50),
2387                     "vnf_id": existing_vnf[0]["uuid"],
2388                     "member_vnf_index": str(vnf["member-vnf-index"]),
2389                     # TODO 'start-by-default': True
2390                 }
2391                 vnf_index2scevnf_uuid[str(vnf['member-vnf-index'])] = sce_vnf_uuid
2392                 vnf_index2vnf_uuid[str(vnf['member-vnf-index'])] = existing_vnf[0]["uuid"]
2393                 db_sce_vnfs.append(db_sce_vnf)
2394
2395             # table ip_profiles (ip-profiles)
2396             ip_profile_name2db_table_index = {}
2397             for ip_profile in nsd.get("ip-profiles").values():
2398                 db_ip_profile = {
2399                     "ip_version": str(ip_profile["ip-profile-params"].get("ip-version", "ipv4")),
2400                     "subnet_address": str(ip_profile["ip-profile-params"].get("subnet-address")),
2401                     "gateway_address": str(ip_profile["ip-profile-params"].get("gateway-address")),
2402                     "dhcp_enabled": str(ip_profile["ip-profile-params"]["dhcp-params"].get("enabled", True)),
2403                     "dhcp_start_address": str(ip_profile["ip-profile-params"]["dhcp-params"].get("start-address")),
2404                     "dhcp_count": str(ip_profile["ip-profile-params"]["dhcp-params"].get("count")),
2405                 }
2406                 dns_list = []
2407                 for dns in ip_profile["ip-profile-params"]["dns-server"].values():
2408                     dns_list.append(str(dns.get("address")))
2409                 db_ip_profile["dns_address"] = ";".join(dns_list)
2410                 if ip_profile["ip-profile-params"].get('security-group'):
2411                     db_ip_profile["security_group"] = ip_profile["ip-profile-params"]['security-group']
2412                 ip_profile_name2db_table_index[str(ip_profile["name"])] = db_ip_profiles_index
2413                 db_ip_profiles_index += 1
2414                 db_ip_profiles.append(db_ip_profile)
2415
2416             # table sce_nets (internal-vld)
2417             for vld in nsd.get("vld").values():
2418                 sce_net_uuid = str(uuid4())
2419                 uuid_list.append(sce_net_uuid)
2420                 db_sce_net = {
2421                     "uuid": sce_net_uuid,
2422                     "name": get_str(vld, "name", 255),
2423                     "scenario_id": scenario_uuid,
2424                     # "type": #TODO
2425                     "multipoint": not vld.get("type") == "ELINE",
2426                     "osm_id":  get_str(vld, "id", 255),
2427                     # "external": #TODO
2428                     "description": get_str(vld, "description", 255),
2429                 }
2430                 # guess type of network
2431                 if vld.get("mgmt-network"):
2432                     db_sce_net["type"] = "bridge"
2433                     db_sce_net["external"] = True
2434                 elif vld.get("provider-network").get("overlay-type") == "VLAN":
2435                     db_sce_net["type"] = "data"
2436                 else:
2437                     # later on it will be fixed to bridge or data depending on the type of interfaces attached to it
2438                     db_sce_net["type"] = None
2439                 db_sce_nets.append(db_sce_net)
2440
2441                 # ip-profile, link db_ip_profile with db_sce_net
2442                 if vld.get("ip-profile-ref"):
2443                     ip_profile_name = vld.get("ip-profile-ref")
2444                     if ip_profile_name not in ip_profile_name2db_table_index:
2445                         raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'ip-profile-ref':'{}'."
2446                                             " Reference to a non-existing 'ip_profiles'".format(
2447                                                 str(nsd["id"]), str(vld["id"]), str(vld["ip-profile-ref"])),
2448                                             httperrors.Bad_Request)
2449                     db_ip_profiles[ip_profile_name2db_table_index[ip_profile_name]]["sce_net_id"] = sce_net_uuid
2450                 elif vld.get("vim-network-name"):
2451                     db_sce_net["vim_network_name"] = get_str(vld, "vim-network-name", 255)
2452
2453                 # table sce_interfaces (vld:vnfd-connection-point-ref)
2454                 for iface in vld.get("vnfd-connection-point-ref").values():
2455                     # Check if there are VDUs in the descriptor
2456                     vnf_index = str(iface['member-vnf-index-ref'])
2457                     existing_vdus = mydb.get_rows(SELECT=('vms.uuid'), FROM="vms", WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index]})
2458                     if existing_vdus:
2459                         # check correct parameters
2460                         if vnf_index not in vnf_index2vnf_uuid:
2461                             raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'vnfd-connection-point"
2462                                               "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at "
2463                                               "'nsd':'constituent-vnfd'".format(
2464                                                   str(nsd["id"]), str(vld["id"]), str(iface["member-vnf-index-ref"])),
2465                                               httperrors.Bad_Request)
2466
2467                         existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid', 'i.type as iface_type'),
2468                                                       FROM="interfaces as i join vms on i.vm_id=vms.uuid",
2469                                                       WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index],
2470                                                              'external_name': get_str(iface, "vnfd-connection-point-ref",
2471                                                                                       255)})
2472                         if not existing_ifaces:
2473                             raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'vnfd-connection-point"
2474                                               "-ref':'vnfd-connection-point-ref':'{}'. Reference to a non-existing "
2475                                               "connection-point name at VNFD '{}'".format(
2476                                                   str(nsd["id"]), str(vld["id"]), str(iface["vnfd-connection-point-ref"]),
2477                                                   str(iface.get("vnfd-id-ref"))[:255]),
2478                                               httperrors.Bad_Request)
2479                         interface_uuid = existing_ifaces[0]["uuid"]
2480                         if existing_ifaces[0]["iface_type"] == "data":
2481                             db_sce_net["type"] = "data"
2482                         sce_interface_uuid = str(uuid4())
2483                         uuid_list.append(sce_net_uuid)
2484                         iface_ip_address = None
2485                         if iface.get("ip-address"):
2486                             iface_ip_address = str(iface.get("ip-address"))
2487                         db_sce_interface = {
2488                             "uuid": sce_interface_uuid,
2489                             "sce_vnf_id": vnf_index2scevnf_uuid[vnf_index],
2490                             "sce_net_id": sce_net_uuid,
2491                             "interface_id": interface_uuid,
2492                             "ip_address": iface_ip_address,
2493                         }
2494                         db_sce_interfaces.append(db_sce_interface)
2495                         if not db_sce_net["type"]:
2496                             db_sce_net["type"] = "bridge"
2497
2498             # table sce_vnffgs (vnffgd)
2499             for vnffg in nsd.get("vnffgd").values():
2500                 sce_vnffg_uuid = str(uuid4())
2501                 uuid_list.append(sce_vnffg_uuid)
2502                 db_sce_vnffg = {
2503                     "uuid": sce_vnffg_uuid,
2504                     "name": get_str(vnffg, "name", 255),
2505                     "scenario_id": scenario_uuid,
2506                     "vendor": get_str(vnffg, "vendor", 255),
2507                     "description": get_str(vld, "description", 255),
2508                 }
2509                 db_sce_vnffgs.append(db_sce_vnffg)
2510
2511                 # deal with rsps
2512                 for rsp in vnffg.get("rsp").values():
2513                     sce_rsp_uuid = str(uuid4())
2514                     uuid_list.append(sce_rsp_uuid)
2515                     db_sce_rsp = {
2516                         "uuid": sce_rsp_uuid,
2517                         "name": get_str(rsp, "name", 255),
2518                         "sce_vnffg_id": sce_vnffg_uuid,
2519                         "id": get_str(rsp, "id", 255), # only useful to link with classifiers; will be removed later in the code
2520                     }
2521                     db_sce_rsps.append(db_sce_rsp)
2522                     for iface in rsp.get("vnfd-connection-point-ref").values():
2523                         vnf_index = str(iface['member-vnf-index-ref'])
2524                         if_order = int(iface['order'])
2525                         # check correct parameters
2526                         if vnf_index not in vnf_index2vnf_uuid:
2527                             raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point"
2528                                                 "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at "
2529                                                 "'nsd':'constituent-vnfd'".format(
2530                                                     str(nsd["id"]), str(rsp["id"]), str(iface["member-vnf-index-ref"])),
2531                                                 httperrors.Bad_Request)
2532
2533                         ingress_existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',),
2534                                                                 FROM="interfaces as i join vms on i.vm_id=vms.uuid",
2535                                                                 WHERE={
2536                                                                     'vnf_id': vnf_index2vnf_uuid[vnf_index],
2537                                                                     'external_name': get_str(iface, "vnfd-ingress-connection-point-ref",
2538                                                                                              255)})
2539                         if not ingress_existing_ifaces:
2540                             raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point"
2541                                                 "-ref':'vnfd-ingress-connection-point-ref':'{}'. Reference to a non-existing "
2542                                                 "connection-point name at VNFD '{}'".format(
2543                                 str(nsd["id"]), str(rsp["id"]), str(iface["vnfd-ingress-connection-point-ref"]),
2544                                 str(iface.get("vnfd-id-ref"))[:255]), httperrors.Bad_Request)
2545
2546                         egress_existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',),
2547                                                                FROM="interfaces as i join vms on i.vm_id=vms.uuid",
2548                                                                WHERE={
2549                                                                    'vnf_id': vnf_index2vnf_uuid[vnf_index],
2550                                                                    'external_name': get_str(iface, "vnfd-egress-connection-point-ref",
2551                                                                                             255)})
2552                         if not egress_existing_ifaces:
2553                             raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point"
2554                                                 "-ref':'vnfd-egress-connection-point-ref':'{}'. Reference to a non-existing "
2555                                                 "connection-point name at VNFD '{}'".format(
2556                                 str(nsd["id"]), str(rsp["id"]), str(iface["vnfd-egress-connection-point-ref"]),
2557                                 str(iface.get("vnfd-id-ref"))[:255]), HTTP_Bad_Request)
2558
2559                         ingress_interface_uuid = ingress_existing_ifaces[0]["uuid"]
2560                         egress_interface_uuid = egress_existing_ifaces[0]["uuid"]
2561                         sce_rsp_hop_uuid = str(uuid4())
2562                         uuid_list.append(sce_rsp_hop_uuid)
2563                         db_sce_rsp_hop = {
2564                             "uuid": sce_rsp_hop_uuid,
2565                             "if_order": if_order,
2566                             "ingress_interface_id": ingress_interface_uuid,
2567                             "egress_interface_id": egress_interface_uuid,
2568                             "sce_vnf_id": vnf_index2scevnf_uuid[vnf_index],
2569                             "sce_rsp_id": sce_rsp_uuid,
2570                         }
2571                         db_sce_rsp_hops.append(db_sce_rsp_hop)
2572
2573                 # deal with classifiers
2574                 for classifier in vnffg.get("classifier").values():
2575                     sce_classifier_uuid = str(uuid4())
2576                     uuid_list.append(sce_classifier_uuid)
2577
2578                     # source VNF
2579                     vnf_index = str(classifier['member-vnf-index-ref'])
2580                     if vnf_index not in vnf_index2vnf_uuid:
2581                         raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'classifier[{}]':'vnfd-connection-point"
2582                                             "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at "
2583                                             "'nsd':'constituent-vnfd'".format(
2584                                                 str(nsd["id"]), str(classifier["id"]), str(classifier["member-vnf-index-ref"])),
2585                                             httperrors.Bad_Request)
2586                     existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',),
2587                                                     FROM="interfaces as i join vms on i.vm_id=vms.uuid",
2588                                                     WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index],
2589                                                            'external_name': get_str(classifier, "vnfd-connection-point-ref",
2590                                                                                     255)})
2591                     if not existing_ifaces:
2592                         raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point"
2593                                             "-ref':'vnfd-connection-point-ref':'{}'. Reference to a non-existing "
2594                                             "connection-point name at VNFD '{}'".format(
2595                                                 str(nsd["id"]), str(rsp["id"]), str(iface["vnfd-connection-point-ref"]),
2596                                                 str(iface.get("vnfd-id-ref"))[:255]),
2597                                             httperrors.Bad_Request)
2598                     interface_uuid = existing_ifaces[0]["uuid"]
2599
2600                     db_sce_classifier = {
2601                         "uuid": sce_classifier_uuid,
2602                         "name": get_str(classifier, "name", 255),
2603                         "sce_vnffg_id": sce_vnffg_uuid,
2604                         "sce_vnf_id": vnf_index2scevnf_uuid[vnf_index],
2605                         "interface_id": interface_uuid,
2606                     }
2607                     rsp_id = get_str(classifier, "rsp-id-ref", 255)
2608                     rsp = next((item for item in db_sce_rsps if item["id"] == rsp_id), None)
2609                     db_sce_classifier["sce_rsp_id"] = rsp["uuid"]
2610                     db_sce_classifiers.append(db_sce_classifier)
2611
2612                     for match in classifier.get("match-attributes").values():
2613                         sce_classifier_match_uuid = str(uuid4())
2614                         uuid_list.append(sce_classifier_match_uuid)
2615                         db_sce_classifier_match = {
2616                             "uuid": sce_classifier_match_uuid,
2617                             "ip_proto": get_str(match, "ip-proto", 2),
2618                             "source_ip": get_str(match, "source-ip-address", 16),
2619                             "destination_ip": get_str(match, "destination-ip-address", 16),
2620                             "source_port": get_str(match, "source-port", 5),
2621                             "destination_port": get_str(match, "destination-port", 5),
2622                             "sce_classifier_id": sce_classifier_uuid,
2623                         }
2624                         db_sce_classifier_matches.append(db_sce_classifier_match)
2625                     # TODO: vnf/cp keys
2626
2627         # remove unneeded id's in sce_rsps
2628         for rsp in db_sce_rsps:
2629             rsp.pop('id')
2630
2631         db_tables = [
2632             {"scenarios": db_scenarios},
2633             {"sce_nets": db_sce_nets},
2634             {"ip_profiles": db_ip_profiles},
2635             {"sce_vnfs": db_sce_vnfs},
2636             {"sce_interfaces": db_sce_interfaces},
2637             {"sce_vnffgs": db_sce_vnffgs},
2638             {"sce_rsps": db_sce_rsps},
2639             {"sce_rsp_hops": db_sce_rsp_hops},
2640             {"sce_classifiers": db_sce_classifiers},
2641             {"sce_classifier_matches": db_sce_classifier_matches},
2642         ]
2643
2644         logger.debug("new_nsd_v3 done: %s",
2645                     yaml.safe_dump(db_tables, indent=4, default_flow_style=False) )
2646         mydb.new_rows(db_tables, uuid_list)
2647         return nsd_uuid_list
2648     except NfvoException:
2649         raise
2650     except Exception as e:
2651         logger.error("Exception {}".format(e))