[osm/RO.git] / RO / osm_ro / nfvo.py

# -*- coding: utf-8 -*-

##
# Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
# This file is part of openmano
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# For those usages not covered by the Apache License, Version 2.0 please
# contact with: nfvlabs@tid.es
##
 
'''
NFVO engine, implementing all the methods for the creation, deletion and management of vnfs, scenarios and instances
'''
__author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes"
__date__ ="$16-sep-2014 22:05:01$"

# import imp
import json
import string
import yaml
from random import choice as random_choice
from osm_ro import utils
from osm_ro.utils import deprecated
from osm_ro.vim_thread import vim_thread
import osm_ro.console_proxy_thread as cli
from osm_ro_plugin.vim_dummy import VimDummyConnector
from osm_ro_plugin.sdn_dummy import SdnDummyConnector
from osm_ro_plugin.sdn_failing import SdnFailingConnector
from osm_ro_plugin import vimconn, sdnconn
import logging
import collections
import math
from uuid import uuid4
from osm_ro.db_base import db_base_Exception

from osm_ro import nfvo_db
from threading import Lock
import time as t
from osm_ro.sdn import Sdn, SdnException as ovimException

from Crypto.PublicKey import RSA

import osm_im.vnfd as vnfd_catalog
import osm_im.nsd as nsd_catalog
from pyangbind.lib.serialise import pybindJSONDecoder
from copy import deepcopy
from pkg_resources import iter_entry_points


# WIM
from .http_tools import errors as httperrors
from .wim.engine import WimEngine
from .wim.persistence import WimPersistence
from copy import deepcopy
from pprint import pformat
#

global global_config
# WIM
global wim_engine
wim_engine  = None
global sdnconn_imported
#
global logger
global default_volume_size
default_volume_size = '5' #size in GB
global ovim
ovim = None
global_config = None

plugins = {}   # dictionary with VIM type as key, loaded module as value
vim_threads = {"running":{}, "deleting": {}, "names": []}      # threads running for attached-VIMs
vim_persistent_info = {}
# WIM
sdnconn_imported = {}   # dictionary with WIM type as key, loaded module as value
wim_threads = {"running":{}, "deleting": {}, "names": []}      # threads running for attached-WIMs
wim_persistent_info = {}
#

logger = logging.getLogger('openmano.nfvo')
task_lock = Lock()
last_task_id = 0.0
db = None
db_lock = Lock()

worker_id = None

class NfvoException(httperrors.HttpMappedError):
    """Common Class for NFVO errors"""

def _load_plugin(name, type="vim"):
    # type can be vim or sdn
    global plugins
    try:
        for v in iter_entry_points('osm_ro{}.plugins'.format(type), name):
            plugins[name] = v.load()
    except Exception as e:
        logger.critical("Cannot load osm_{}: {}".format(name, e))
        if name:
            plugins[name] = SdnFailingConnector("Cannot load osm_{}: {}".format(name, e))
    if name and name not in plugins:
        error_text = "Cannot load a module for {t} type '{n}'. The plugin 'osm_{n}' has not been" \
                     " registered".format(t=type, n=name)
        logger.critical(error_text)
        plugins[name] = SdnFailingConnector(error_text)
        # raise NfvoException("Cannot load a module for {t} type '{n}'. The plugin 'osm_{n}' has not been registered".
        #                     format(t=type, n=name), httperrors.Bad_Request)

def get_task_id():
    global last_task_id
    task_id = t.time()
    if task_id <= last_task_id:
        task_id = last_task_id + 0.000001
    last_task_id = task_id
    return "ACTION-{:.6f}".format(task_id)
    # return (t.strftime("%Y%m%dT%H%M%S.{}%Z", t.localtime(task_id))).format(int((task_id % 1)*1e6))


def new_task(name, params, depends=None):
    """Deprected!!!"""
    task_id = get_task_id()
    task = {"status": "enqueued", "id": task_id, "name": name, "params": params}
    if depends:
        task["depends"] = depends
    return task


def is_task_id(id):
    return True if id[:5] == "TASK-" else False

def get_process_id():
    """
    Obtain a unique ID for this process. If running from inside docker, it will get docker ID. If not it
    will provide a random one
    :return: Obtained ID
    """
    # Try getting docker id. If fails, get pid
    try:
        with open("/proc/self/cgroup", "r") as f:
            for text_id_ in f.readlines():
                if "docker/" not in text_id_:
                    continue
                _, _, text_id = text_id_.rpartition("/")
                text_id = text_id.replace("\n", "")[:12]
                if text_id:
                    return text_id
    except Exception:
        pass
    # Return a random id
    return "".join(random_choice("0123456789abcdef") for _ in range(12))

def get_non_used_vim_name(datacenter_name, datacenter_id):
    return "{}:{}:{}".format(
        worker_id[:12], datacenter_id.replace("-", "")[:32], datacenter_name[:16]
    )

# -- Move
def get_non_used_wim_name(wim_name, wim_id, tenant_name, tenant_id):
    name = wim_name[:16]
    if name not in wim_threads["names"]:
        wim_threads["names"].append(name)
        return name
    name = wim_name[:16] + "." + tenant_name[:16]
    if name not in wim_threads["names"]:
        wim_threads["names"].append(name)
        return name
    name = wim_id + "-" + tenant_id
    wim_threads["names"].append(name)
    return name


def start_service(mydb, persistence=None, wim=None):
    global db, global_config, plugins, ovim, worker_id
    db = nfvo_db.nfvo_db(lock=db_lock)
    mydb.lock = db_lock
    db.connect(global_config['db_host'], global_config['db_user'], global_config['db_passwd'], global_config['db_name'])

    persistence = persistence or  WimPersistence(db)

    try:
        worker_id = get_process_id()
        if "rosdn_dummy" not in plugins:
            plugins["rosdn_dummy"] = SdnDummyConnector
        if "rovim_dummy" not in plugins:
            plugins["rovim_dummy"] = VimDummyConnector
        # starts ovim library
        ovim = Sdn(db, plugins)

        global wim_engine
        wim_engine = wim or WimEngine(persistence, plugins)
        wim_engine.ovim = ovim

        ovim.start_service()

        #delete old unneeded vim_wim_actions
        clean_db(mydb)

        # starts vim_threads
        from_= 'tenants_datacenters as td join datacenters as d on td.datacenter_id=d.uuid join '\
                'datacenter_tenants as dt on td.datacenter_tenant_id=dt.uuid'
        select_ = ('type', 'd.config as config', 'd.uuid as datacenter_id', 'vim_url', 'vim_url_admin',
                   'd.name as datacenter_name', 'dt.uuid as datacenter_tenant_id',
                   'dt.vim_tenant_name as vim_tenant_name', 'dt.vim_tenant_id as vim_tenant_id',
                   'user', 'passwd', 'dt.config as dt_config', 'nfvo_tenant_id')
        vims = mydb.get_rows(FROM=from_, SELECT=select_)
        for vim in vims:
            extra={'datacenter_tenant_id': vim.get('datacenter_tenant_id'),
                   'datacenter_id': vim.get('datacenter_id')}
            if vim["config"]:
                extra.update(yaml.load(vim["config"], Loader=yaml.Loader))
            if vim.get('dt_config'):
                extra.update(yaml.load(vim["dt_config"], Loader=yaml.Loader))
            plugin_name = "rovim_" + vim["type"]
            if plugin_name not in plugins:
                _load_plugin(plugin_name, type="vim")

            thread_id = vim['datacenter_tenant_id']
            vim_persistent_info[thread_id] = {}
            try:
                #if not tenant:
                #    return -httperrors.Bad_Request, "You must provide a valid tenant name or uuid for VIM  %s" % ( vim["type"])
                myvim = plugins[plugin_name](
                    uuid=vim['datacenter_id'], name=vim['datacenter_name'],
                    tenant_id=vim['vim_tenant_id'], tenant_name=vim['vim_tenant_name'],
                    url=vim['vim_url'], url_admin=vim['vim_url_admin'],
                    user=vim['user'], passwd=vim['passwd'],
                    config=extra, persistent_info=vim_persistent_info[thread_id]
                )
            except vimconn.VimConnException as e:
                myvim = e
                logger.error("Cannot launch thread for VIM {} '{}': {}".format(vim['datacenter_name'],
                                                                               vim['datacenter_id'], e))
            except Exception as e:
                logger.critical("Cannot launch thread for VIM {} '{}': {}".format(vim['datacenter_name'],
                                                                                  vim['datacenter_id'], e))
                # raise NfvoException("Error at VIM  {}; {}: {}".format(vim["type"], type(e).__name__, e),
                #                     httperrors.Internal_Server_Error)
            thread_name = get_non_used_vim_name(vim['datacenter_name'], vim['datacenter_id'])
            new_thread = vim_thread(task_lock, plugins, thread_name, None,
                                    vim['datacenter_tenant_id'], db=db)
            new_thread.start()
            vim_threads["running"][thread_id] = new_thread
        wims = mydb.get_rows(FROM="wim_accounts join wims on wim_accounts.wim_id=wims.uuid",
                             WHERE={"sdn": "true"},
                             SELECT=("wim_accounts.uuid as uuid", "type", "wim_accounts.name as name"))
        for wim in wims:
            plugin_name = "rosdn_" + wim["type"]
            if plugin_name not in plugins:
                _load_plugin(plugin_name, type="sdn")

            thread_id = wim['uuid']
            thread_name = get_non_used_vim_name(wim['name'], wim['uuid'])
            new_thread = vim_thread(task_lock, plugins, thread_name, wim['uuid'], None, db=db)
            new_thread.start()
            vim_threads["running"][thread_id] = new_thread
        wim_engine.start_threads()
    except db_base_Exception as e:
        raise NfvoException(str(e) + " at nfvo.get_vim", e.http_code)
    except ovimException as e:
        message = str(e)
        if message[:22] == "DATABASE wrong version":
            message = "DATABASE wrong version of lib_osm_openvim {msg} -d{dbname} -u{dbuser} -p{dbpass} {ver}' "\
                      "at host {dbhost}".format(
                            msg=message[22:-3], dbname=global_config["db_ovim_name"],
                            dbuser=global_config["db_ovim_user"], dbpass=global_config["db_ovim_passwd"],
                            ver=message[-3:-1], dbhost=global_config["db_ovim_host"])
        raise NfvoException(message, httperrors.Bad_Request)


def stop_service():
    global ovim, global_config
    if ovim:
        ovim.stop_service()
    for thread_id, thread in vim_threads["running"].items():
        thread.insert_task("exit")
        vim_threads["deleting"][thread_id] = thread
    vim_threads["running"] = {}

    if wim_engine:
        wim_engine.stop_threads()

    if global_config and global_config.get("console_thread"):
        for thread in global_config["console_thread"]:
            thread.terminate = True

def get_version():
    return  ("openmanod version {} {}\n(c) Copyright Telefonica".format(global_config["version"],
                                                                        global_config["version_date"] ))

def clean_db(mydb):
    """
    Clean unused or old entries at database to avoid unlimited growing
    :param mydb: database connector
    :return: None
    """
    # get and delete unused vim_wim_actions: all elements deleted, one week before, instance not present
    now = t.time()-3600*24*7
    instance_action_id = None
    nb_deleted = 0
    while True:
        actions_to_delete = mydb.get_rows(
            SELECT=("item", "item_id", "instance_action_id"),
            FROM="vim_wim_actions as va join instance_actions as ia on va.instance_action_id=ia.uuid "
                    "left join instance_scenarios as i on ia.instance_id=i.uuid",
            WHERE={"va.action": "DELETE", "va.modified_at<": now, "i.uuid": None,
                   "va.status": ("DONE", "SUPERSEDED")},
            LIMIT=100
        )
        for to_delete in actions_to_delete:
            mydb.delete_row(FROM="vim_wim_actions", WHERE=to_delete)
            if instance_action_id != to_delete["instance_action_id"]:
                instance_action_id = to_delete["instance_action_id"]
                mydb.delete_row(FROM="instance_actions", WHERE={"uuid": instance_action_id})
        nb_deleted += len(actions_to_delete)
        if len(actions_to_delete) < 100:
            break
    # clean locks
    mydb.update_rows("vim_wim_actions", UPDATE={"worker": None}, WHERE={"worker<>": None})

    if nb_deleted:
        logger.debug("Removed {} unused vim_wim_actions".format(nb_deleted))


def get_flavorlist(mydb, vnf_id, nfvo_tenant=None):
    '''Obtain flavorList
    return result, content:
        <0, error_text upon error
        nb_records, flavor_list on success
    '''
    WHERE_dict={}
    WHERE_dict['vnf_id'] = vnf_id
    if nfvo_tenant is not None:
        WHERE_dict['nfvo_tenant_id'] = nfvo_tenant

    #result, content = mydb.get_table(FROM='vms join vnfs on vms.vnf_id = vnfs.uuid',SELECT=('uuid'),WHERE=WHERE_dict )
    #result, content = mydb.get_table(FROM='vms',SELECT=('vim_flavor_id',),WHERE=WHERE_dict )
    flavors = mydb.get_rows(FROM='vms join flavors on vms.flavor_id=flavors.uuid',SELECT=('flavor_id',),WHERE=WHERE_dict )
    #print "get_flavor_list result:", result
    #print "get_flavor_list content:", content
    flavorList=[]
    for flavor in flavors:
        flavorList.append(flavor['flavor_id'])
    return flavorList


def get_imagelist(mydb, vnf_id, nfvo_tenant=None):
    """
    Get used images of all vms belonging to this VNFD
    :param mydb: database conector
    :param vnf_id: vnfd uuid
    :param nfvo_tenant: tenant, not used
    :return: The list of image uuid used
    """
    image_list = []
    vms = mydb.get_rows(SELECT=('image_id','image_list'), FROM='vms', WHERE={'vnf_id': vnf_id})
    for vm in vms:
        if vm["image_id"] and vm["image_id"] not in image_list:
            image_list.append(vm["image_id"])
        if vm["image_list"]:
            vm_image_list = yaml.load(vm["image_list"], Loader=yaml.Loader)
            for image_dict in vm_image_list:
                if image_dict["image_id"] not in image_list:
                    image_list.append(image_dict["image_id"])
    return image_list


def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, datacenter_tenant_id=None,
            vim_tenant=None, vim_tenant_name=None, vim_user=None, vim_passwd=None, ignore_errors=False):
    '''Obtain a dictionary of VIM (datacenter) classes with some of the input parameters
    return dictionary with {datacenter_id: vim_class, ... }. vim_class contain:
            'nfvo_tenant_id','datacenter_id','vim_tenant_id','vim_url','vim_url_admin','datacenter_name','type','user','passwd'
        raise exception upon error
    '''
    global plugins
    WHERE_dict={}
    if nfvo_tenant     is not None:  WHERE_dict['nfvo_tenant_id'] = nfvo_tenant
    if datacenter_id   is not None:  WHERE_dict['d.uuid']  = datacenter_id
    if datacenter_tenant_id is not None:  WHERE_dict['datacenter_tenant_id']  = datacenter_tenant_id
    if datacenter_name is not None:  WHERE_dict['d.name']  = datacenter_name
    if vim_tenant      is not None:  WHERE_dict['dt.vim_tenant_id']  = vim_tenant
    if vim_tenant_name is not None:  WHERE_dict['vim_tenant_name']  = vim_tenant_name
    if nfvo_tenant or vim_tenant or vim_tenant_name or datacenter_tenant_id:
        from_= 'tenants_datacenters as td join datacenters as d on td.datacenter_id=d.uuid join datacenter_tenants as dt on td.datacenter_tenant_id=dt.uuid'
        select_ = ('type','d.config as config','d.uuid as datacenter_id', 'vim_url', 'vim_url_admin', 'd.name as datacenter_name',
                   'dt.uuid as datacenter_tenant_id','dt.vim_tenant_name as vim_tenant_name','dt.vim_tenant_id as vim_tenant_id',
                   'user','passwd', 'dt.config as dt_config')
    else:
        from_ = 'datacenters as d'
        select_ = ('type','config','d.uuid as datacenter_id', 'vim_url', 'vim_url_admin', 'd.name as datacenter_name')
    try:
        vims = mydb.get_rows(FROM=from_, SELECT=select_, WHERE=WHERE_dict )
        vim_dict={}
        for vim in vims:
            extra={'datacenter_tenant_id': vim.get('datacenter_tenant_id'),
                   'datacenter_id': vim.get('datacenter_id'),
                   '_vim_type_internal': vim.get('type')}
            if vim["config"]:
                extra.update(yaml.load(vim["config"], Loader=yaml.Loader))
            if vim.get('dt_config'):
                extra.update(yaml.load(vim["dt_config"], Loader=yaml.Loader))
            plugin_name = "rovim_" + vim["type"]
            if plugin_name not in plugins:
                try:
                    _load_plugin(plugin_name, type="vim")
                except NfvoException as e:
                    if ignore_errors:
                        logger.error("{}".format(e))
                        continue
                    else:
                        raise
            try:
                if 'datacenter_tenant_id' in vim:
                    thread_id = vim["datacenter_tenant_id"]
                    if thread_id not in vim_persistent_info:
                        vim_persistent_info[thread_id] = {}
                    persistent_info = vim_persistent_info[thread_id]
                else:
                    persistent_info = {}
                #if not tenant:
                #    return -httperrors.Bad_Request, "You must provide a valid tenant name or uuid for VIM  %s" % ( vim["type"])
                vim_dict[vim['datacenter_id']] = plugins[plugin_name](
                                uuid=vim['datacenter_id'], name=vim['datacenter_name'],
                                tenant_id=vim.get('vim_tenant_id',vim_tenant),
                                tenant_name=vim.get('vim_tenant_name',vim_tenant_name),
                                url=vim['vim_url'], url_admin=vim['vim_url_admin'],
                                user=vim.get('user',vim_user), passwd=vim.get('passwd',vim_passwd),
                                config=extra, persistent_info=persistent_info
                        )
            except Exception as e:
                if ignore_errors:
                    logger.error("Error at VIM  {}; {}: {}".format(vim["type"], type(e).__name__, str(e)))
                    continue
                http_code = httperrors.Internal_Server_Error
                if isinstance(e, vimconn.VimConnException):
                    http_code = e.http_code
                raise NfvoException("Error at VIM  {}; {}: {}".format(vim["type"], type(e).__name__, str(e)), http_code)
        return vim_dict
    except db_base_Exception as e:
        raise NfvoException(str(e) + " at nfvo.get_vim", e.http_code)


def rollback(mydb,  vims, rollback_list):
    undeleted_items=[]
    #delete things by reverse order
    for i in range(len(rollback_list)-1, -1, -1):
        item = rollback_list[i]
        if item["where"]=="vim":
            if item["vim_id"] not in vims:
                continue
            if is_task_id(item["uuid"]):
                continue
            vim = vims[item["vim_id"]]
            try:
                if item["what"]=="image":
                    vim.delete_image(item["uuid"])
                    mydb.delete_row(FROM="datacenters_images", WHERE={"datacenter_vim_id": vim["id"], "vim_id":item["uuid"]})
                elif item["what"]=="flavor":
                    vim.delete_flavor(item["uuid"])
                    mydb.delete_row(FROM="datacenters_flavors", WHERE={"datacenter_vim_id": vim["id"], "vim_id":item["uuid"]})
                elif item["what"]=="network":
                    vim.delete_network(item["uuid"])
                elif item["what"]=="vm":
                    vim.delete_vminstance(item["uuid"])
            except vimconn.VimConnException as e:
                logger.error("Error in rollback. Not possible to delete VIM %s '%s'. Message: %s", item['what'], item["uuid"], str(e))
                undeleted_items.append("{} {} from VIM {}".format(item['what'], item["uuid"], vim["name"]))
            except db_base_Exception as e:
                logger.error("Error in rollback. Not possible to delete %s '%s' from DB.datacenters Message: %s", item['what'], item["uuid"], str(e))

        else: # where==mano
            try:
                if item["what"]=="image":
                    mydb.delete_row(FROM="images", WHERE={"uuid": item["uuid"]})
                elif item["what"]=="flavor":
                    mydb.delete_row(FROM="flavors", WHERE={"uuid": item["uuid"]})
            except db_base_Exception as e:
                logger.error("Error in rollback. Not possible to delete %s '%s' from DB. Message: %s", item['what'], item["uuid"], str(e))
                undeleted_items.append("{} '{}'".format(item['what'], item["uuid"]))
    if len(undeleted_items)==0:
        return True, "Rollback successful."
    else:
        return False, "Rollback fails to delete: " + str(undeleted_items)


def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1):
    global global_config
    #create a dictionary with vnfc-name: vnfc:interface-list  key:values pairs
    vnfc_interfaces={}
    for vnfc in vnf_descriptor["vnf"]["VNFC"]:
        name_dict = {}
        #dataplane interfaces
        for numa in vnfc.get("numas",() ):
            for interface in numa.get("interfaces",()):
                if interface["name"] in name_dict:
                    raise NfvoException(
                        "Error at vnf:VNFC[name:'{}']:numas:interfaces:name, interface name '{}' already used in this VNFC".format(
                            vnfc["name"], interface["name"]),
                        httperrors.Bad_Request)
                name_dict[ interface["name"] ] = "underlay"
        #bridge interfaces
        for interface in vnfc.get("bridge-ifaces",() ):
            if interface["name"] in name_dict:
                raise NfvoException(
                    "Error at vnf:VNFC[name:'{}']:bridge-ifaces:name, interface name '{}' already used in this VNFC".format(
                        vnfc["name"], interface["name"]),
                    httperrors.Bad_Request)
            name_dict[ interface["name"] ] = "overlay"
        vnfc_interfaces[ vnfc["name"] ] = name_dict
        # check bood-data info
        # if "boot-data" in vnfc:
        #     # check that user-data is incompatible with users and config-files
        #     if (vnfc["boot-data"].get("users") or vnfc["boot-data"].get("config-files")) and vnfc["boot-data"].get("user-data"):
        #         raise NfvoException(
        #             "Error at vnf:VNFC:boot-data, fields 'users' and 'config-files' are not compatible with 'user-data'",
        #             httperrors.Bad_Request)

    #check if the info in external_connections matches with the one in the vnfcs
    name_list=[]
    for external_connection in vnf_descriptor["vnf"].get("external-connections",() ):
        if external_connection["name"] in name_list:
            raise NfvoException(
                "Error at vnf:external-connections:name, value '{}' already used as an external-connection".format(
                    external_connection["name"]),
                httperrors.Bad_Request)
        name_list.append(external_connection["name"])
        if external_connection["VNFC"] not in vnfc_interfaces:
            raise NfvoException(
                "Error at vnf:external-connections[name:'{}']:VNFC, value '{}' does not match any VNFC".format(
                    external_connection["name"], external_connection["VNFC"]),
                httperrors.Bad_Request)

        if external_connection["local_iface_name"] not in vnfc_interfaces[ external_connection["VNFC"] ]:
            raise NfvoException(
                "Error at vnf:external-connections[name:'{}']:local_iface_name, value '{}' does not match any interface of this VNFC".format(
                    external_connection["name"],
                    external_connection["local_iface_name"]),
                httperrors.Bad_Request )

    #check if the info in internal_connections matches with the one in the vnfcs
    name_list=[]
    for internal_connection in vnf_descriptor["vnf"].get("internal-connections",() ):
        if internal_connection["name"] in name_list:
            raise NfvoException(
                "Error at vnf:internal-connections:name, value '{}' already used as an internal-connection".format(
                    internal_connection["name"]),
                httperrors.Bad_Request)
        name_list.append(internal_connection["name"])
        #We should check that internal-connections of type "ptp" have only 2 elements

        if len(internal_connection["elements"])>2 and (internal_connection.get("type") == "ptp" or internal_connection.get("type") == "e-line"):
            raise NfvoException(
                "Error at 'vnf:internal-connections[name:'{}']:elements', size must be 2 for a '{}' type. Consider change it to '{}' type".format(
                    internal_connection["name"],
                    'ptp' if vnf_descriptor_version==1 else 'e-line',
                    'data' if vnf_descriptor_version==1 else "e-lan"),
                httperrors.Bad_Request)
        for port in internal_connection["elements"]:
            vnf = port["VNFC"]
            iface = port["local_iface_name"]
            if vnf not in vnfc_interfaces:
                raise NfvoException(
                    "Error at vnf:internal-connections[name:'{}']:elements[]:VNFC, value '{}' does not match any VNFC".format(
                        internal_connection["name"], vnf),
                    httperrors.Bad_Request)
            if iface not in vnfc_interfaces[ vnf ]:
                raise NfvoException(
                    "Error at vnf:internal-connections[name:'{}']:elements[]:local_iface_name, value '{}' does not match any interface of this VNFC".format(
                        internal_connection["name"], iface),
                    httperrors.Bad_Request)
                return -httperrors.Bad_Request,
            if vnf_descriptor_version==1 and "type" not in internal_connection:
                if vnfc_interfaces[vnf][iface] == "overlay":
                    internal_connection["type"] = "bridge"
                else:
                    internal_connection["type"] = "data"
            if vnf_descriptor_version==2 and "implementation" not in internal_connection:
                if vnfc_interfaces[vnf][iface] == "overlay":
                    internal_connection["implementation"] = "overlay"
                else:
                    internal_connection["implementation"] = "underlay"
            if (internal_connection.get("type") == "data" or internal_connection.get("type") == "ptp" or \
                internal_connection.get("implementation") == "underlay") and vnfc_interfaces[vnf][iface] == "overlay":
                raise NfvoException(
                    "Error at vnf:internal-connections[name:'{}']:elements[]:{}, interface of type {} connected to an {} network".format(
                        internal_connection["name"],
                        iface, 'bridge' if vnf_descriptor_version==1 else 'overlay',
                        'data' if vnf_descriptor_version==1 else 'underlay'),
                    httperrors.Bad_Request)
            if (internal_connection.get("type") == "bridge" or internal_connection.get("implementation") == "overlay") and \
                vnfc_interfaces[vnf][iface] == "underlay":
                raise NfvoException(
                    "Error at vnf:internal-connections[name:'{}']:elements[]:{}, interface of type {} connected to an {} network".format(
                        internal_connection["name"], iface,
                        'data' if vnf_descriptor_version==1 else 'underlay',
                        'bridge' if vnf_descriptor_version==1 else 'overlay'),
                    httperrors.Bad_Request)


def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=False, return_on_error=None):
    #look if image exist
    if only_create_at_vim:
        image_mano_id = image_dict['uuid']
        if return_on_error == None:
            return_on_error = True
    else:
        if image_dict['location']:
            images = mydb.get_rows(FROM="images", WHERE={'location':image_dict['location'], 'metadata':image_dict['metadata']})
        else:
            images = mydb.get_rows(FROM="images", WHERE={'universal_name':image_dict['universal_name'], 'checksum':image_dict['checksum']})
        if len(images)>=1:
            image_mano_id = images[0]['uuid']
        else:
            #create image in MANO DB
            temp_image_dict={'name':image_dict['name'],         'description':image_dict.get('description',None),
                            'location':image_dict['location'],  'metadata':image_dict.get('metadata',None),
                            'universal_name':image_dict['universal_name'] , 'checksum':image_dict['checksum']
                            }
            #temp_image_dict['location'] = image_dict.get('new_location') if image_dict['location'] is None
            image_mano_id = mydb.new_row('images', temp_image_dict, add_uuid=True)
            rollback_list.append({"where":"mano", "what":"image","uuid":image_mano_id})
    #create image at every vim
    for vim_id,vim in vims.items():
        datacenter_vim_id = vim["config"]["datacenter_tenant_id"]
        image_created="false"
        #look at database
        image_db = mydb.get_rows(FROM="datacenters_images",
                                 WHERE={'datacenter_vim_id': datacenter_vim_id, 'image_id': image_mano_id})
        #look at VIM if this image exist
        try:
            if image_dict['location'] is not None:
                image_vim_id = vim.get_image_id_from_path(image_dict['location'])
            else:
                filter_dict = {}
                filter_dict['name'] = image_dict['universal_name']
                if image_dict.get('checksum') != None:
                    filter_dict['checksum'] = image_dict['checksum']
                #logger.debug('>>>>>>>> Filter dict: %s', str(filter_dict))
                vim_images = vim.get_image_list(filter_dict)
                #logger.debug('>>>>>>>> VIM images: %s', str(vim_images))
                if len(vim_images) > 1:
                    raise vimconn.VimConnException("More than one candidate VIM image found for filter: {}".format(str(filter_dict)), httperrors.Conflict)
                elif len(vim_images) == 0:
                    raise vimconn.VimConnNotFoundException("Image not found at VIM with filter: '{}'".format(str(filter_dict)))
                else:
                    #logger.debug('>>>>>>>> VIM image 0: %s', str(vim_images[0]))
                    image_vim_id = vim_images[0]['id']

        except vimconn.VimConnNotFoundException as e:
            #Create the image in VIM only if image_dict['location'] or image_dict['new_location'] is not None
            try:
                #image_dict['location']=image_dict.get('new_location') if image_dict['location'] is None
                if image_dict['location']:
                    image_vim_id = vim.new_image(image_dict)
                    rollback_list.append({"where":"vim", "vim_id": vim_id, "what":"image","uuid":image_vim_id})
                    image_created="true"
                else:
                    #If we reach this point, then the image has image name, and optionally checksum, and could not be found
                    raise vimconn.VimConnException(str(e))
            except vimconn.VimConnException as e:
                if return_on_error:
                    logger.error("Error creating image at VIM '%s': %s", vim["name"], str(e))
                    raise
                image_vim_id = None
                logger.warn("Error creating image at VIM '%s': %s", vim["name"], str(e))
                continue
        except vimconn.VimConnException as e:
            if return_on_error:
                logger.error("Error contacting VIM to know if the image exists at VIM: %s", str(e))
                raise
            logger.warn("Error contacting VIM to know if the image exists at VIM: %s", str(e))
            image_vim_id = None
            continue
        #if we reach here, the image has been created or existed
        if len(image_db)==0:
            #add new vim_id at datacenters_images
            mydb.new_row('datacenters_images', {'datacenter_vim_id': datacenter_vim_id,
                                                'image_id':image_mano_id,
                                                'vim_id': image_vim_id,
                                                'created':image_created})
        elif image_db[0]["vim_id"]!=image_vim_id:
            #modify existing vim_id at datacenters_images
            mydb.update_rows('datacenters_images', UPDATE={'vim_id':image_vim_id}, WHERE={'datacenter_vim_id':vim_id, 'image_id':image_mano_id})

    return image_vim_id if only_create_at_vim else image_mano_id


def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_vim=False, return_on_error = None):
    temp_flavor_dict= {'disk':flavor_dict.get('disk',0),
            'ram':flavor_dict.get('ram'),
            'vcpus':flavor_dict.get('vcpus'),
        }
    if 'extended' in flavor_dict and flavor_dict['extended']==None:
        del flavor_dict['extended']
    if 'extended' in flavor_dict:
        temp_flavor_dict['extended']=yaml.safe_dump(flavor_dict['extended'],default_flow_style=True,width=256)

    #look if flavor exist
    if only_create_at_vim:
        flavor_mano_id = flavor_dict['uuid']
        if return_on_error == None:
            return_on_error = True
    else:
        flavors = mydb.get_rows(FROM="flavors", WHERE=temp_flavor_dict)
        if len(flavors)>=1:
            flavor_mano_id = flavors[0]['uuid']
        else:
            #create flavor
            #create one by one the images of aditional disks
            dev_image_list=[] #list of images
            if 'extended' in flavor_dict and flavor_dict['extended']!=None:
                dev_nb=0
                for device in flavor_dict['extended'].get('devices',[]):
                    if "image" not in device and "image name" not in device:
                        continue
                    image_dict={}
                    image_dict['name']=device.get('image name',flavor_dict['name']+str(dev_nb)+"-img")
                    image_dict['universal_name']=device.get('image name')
                    image_dict['description']=flavor_dict['name']+str(dev_nb)+"-img"
                    image_dict['location']=device.get('image')
                    #image_dict['new_location']=vnfc.get('image location')
                    image_dict['checksum']=device.get('image checksum')
                    image_metadata_dict = device.get('image metadata', None)
                    image_metadata_str = None
                    if image_metadata_dict != None:
                        image_metadata_str = yaml.safe_dump(image_metadata_dict,default_flow_style=True,width=256)
                    image_dict['metadata']=image_metadata_str
                    image_id = create_or_use_image(mydb, vims, image_dict, rollback_list)
                    #print "Additional disk image id for VNFC %s: %s" % (flavor_dict['name']+str(dev_nb)+"-img", image_id)
                    dev_image_list.append(image_id)
                    dev_nb += 1
            temp_flavor_dict['name'] = flavor_dict['name']
            temp_flavor_dict['description'] = flavor_dict.get('description',None)
            content = mydb.new_row('flavors', temp_flavor_dict, add_uuid=True)
            flavor_mano_id= content
            rollback_list.append({"where":"mano", "what":"flavor","uuid":flavor_mano_id})
    #create flavor at every vim
    if 'uuid' in flavor_dict:
        del flavor_dict['uuid']
    flavor_vim_id=None
    for vim_id,vim in vims.items():
        datacenter_vim_id = vim["config"]["datacenter_tenant_id"]
        flavor_created="false"
        #look at database
        flavor_db = mydb.get_rows(FROM="datacenters_flavors",
                                  WHERE={'datacenter_vim_id': datacenter_vim_id, 'flavor_id': flavor_mano_id})
        #look at VIM if this flavor exist  SKIPPED
        #res_vim, flavor_vim_id = vim.get_flavor_id_from_path(flavor_dict['location'])
        #if res_vim < 0:
        #    print "Error contacting VIM to know if the flavor %s existed previously." %flavor_vim_id
        #    continue
        #elif res_vim==0:

        # Create the flavor in VIM
        # Translate images at devices from MANO id to VIM id
        disk_list = []
        if 'extended' in flavor_dict and flavor_dict['extended']!=None and "devices" in flavor_dict['extended']:
            # make a copy of original devices
            devices_original=[]

            for device in flavor_dict["extended"].get("devices",[]):
                dev={}
                dev.update(device)
                devices_original.append(dev)
                if 'image' in device:
                    del device['image']
                if 'image metadata' in device:
                    del device['image metadata']
                if 'image checksum' in device:
                    del device['image checksum']
            dev_nb = 0
            for index in range(0,len(devices_original)) :
                device=devices_original[index]
                if "image" not in device and "image name" not in device:
                    # if 'size' in device:
                    disk_list.append({'size': device.get('size', default_volume_size), 'name': device.get('name')})
                    continue
                image_dict={}
                image_dict['name']=device.get('image name',flavor_dict['name']+str(dev_nb)+"-img")
                image_dict['universal_name']=device.get('image name')
                image_dict['description']=flavor_dict['name']+str(dev_nb)+"-img"
                image_dict['location']=device.get('image')
                # image_dict['new_location']=device.get('image location')
                image_dict['checksum']=device.get('image checksum')
                image_metadata_dict = device.get('image metadata', None)
                image_metadata_str = None
                if image_metadata_dict != None:
                    image_metadata_str = yaml.safe_dump(image_metadata_dict,default_flow_style=True,width=256)
                image_dict['metadata']=image_metadata_str
                image_mano_id=create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=False, return_on_error=return_on_error )
                image_dict["uuid"]=image_mano_id
                image_vim_id=create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=True, return_on_error=return_on_error)

                #save disk information (image must be based on and size
                disk_list.append({'image_id': image_vim_id, 'size': device.get('size', default_volume_size)})

                flavor_dict["extended"]["devices"][index]['imageRef']=image_vim_id
                dev_nb += 1
        if len(flavor_db)>0:
            #check that this vim_id exist in VIM, if not create
            flavor_vim_id=flavor_db[0]["vim_id"]
            try:
                vim.get_flavor(flavor_vim_id)
                continue #flavor exist
            except vimconn.VimConnException:
                pass
        #create flavor at vim
        logger.debug("nfvo.create_or_use_flavor() adding flavor to VIM %s", vim["name"])
        try:
            flavor_vim_id = None
            flavor_vim_id=vim.get_flavor_id_from_data(flavor_dict)
            flavor_created="false"
        except vimconn.VimConnException as e:
            pass
        try:
            if not flavor_vim_id:
                flavor_vim_id = vim.new_flavor(flavor_dict)
                rollback_list.append({"where":"vim", "vim_id": vim_id, "what":"flavor","uuid":flavor_vim_id})
                flavor_created="true"
        except vimconn.VimConnException as e:
            if return_on_error:
                logger.error("Error creating flavor at VIM %s: %s.", vim["name"], str(e))
                raise
            logger.warn("Error creating flavor at VIM %s: %s.", vim["name"], str(e))
            flavor_vim_id = None
            continue
        #if reach here the flavor has been create or exist
        if len(flavor_db)==0:
            #add new vim_id at datacenters_flavors
            extended_devices_yaml = None
            if len(disk_list) > 0:
                extended_devices = dict()
                extended_devices['disks'] = disk_list
                extended_devices_yaml = yaml.safe_dump(extended_devices,default_flow_style=True,width=256)
            mydb.new_row('datacenters_flavors',
                        {'datacenter_vim_id': datacenter_vim_id, 'flavor_id': flavor_mano_id, 'vim_id': flavor_vim_id,
                        'created': flavor_created, 'extended': extended_devices_yaml})
        elif flavor_db[0]["vim_id"]!=flavor_vim_id:
            #modify existing vim_id at datacenters_flavors
            mydb.update_rows('datacenters_flavors', UPDATE={'vim_id':flavor_vim_id},
                             WHERE={'datacenter_vim_id': datacenter_vim_id, 'flavor_id': flavor_mano_id})

    return flavor_vim_id if only_create_at_vim else flavor_mano_id


def get_str(obj, field, length):
    """
    Obtain the str value,
    :param obj:
    :param length:
    :return:
    """
    value = obj.get(field)
    if value is not None:
        value = str(value)[:length]
    return value

def _lookfor_or_create_image(db_image, mydb, descriptor):
    """
    fill image content at db_image dictionary. Check if the image with this image and checksum exist
    :param db_image: dictionary to insert data
    :param mydb: database connector
    :param descriptor: yang descriptor
    :return: uuid if the image exist at DB, or None if a new image must be created with the data filled at db_image
    """

    db_image["name"] = get_str(descriptor, "image", 255)
    db_image["checksum"] = get_str(descriptor, "image-checksum", 32)
    if not db_image["checksum"]:  # Ensure that if empty string, None is stored
        db_image["checksum"] = None
    if db_image["name"].startswith("/"):
        db_image["location"] = db_image["name"]
        existing_images = mydb.get_rows(FROM="images", WHERE={'location': db_image["location"]})
    else:
        db_image["universal_name"] = db_image["name"]
        existing_images = mydb.get_rows(FROM="images", WHERE={'universal_name': db_image['universal_name'],
                                                              'checksum': db_image['checksum']})
    if existing_images:
        return existing_images[0]["uuid"]
    else:
        image_uuid = str(uuid4())
        db_image["uuid"] = image_uuid
        return None

def get_resource_allocation_params(quota_descriptor):
    """
    read the quota_descriptor from vnfd and fetch the resource allocation properties from the descriptor object
    :param quota_descriptor: cpu/mem/vif/disk-io quota descriptor
    :return: quota params for limit, reserve, shares from the descriptor object
    """
    quota = {}
    if quota_descriptor.get("limit"):
        quota["limit"] = int(quota_descriptor["limit"])
    if quota_descriptor.get("reserve"):
        quota["reserve"] = int(quota_descriptor["reserve"])
    if quota_descriptor.get("shares"):
        quota["shares"] = int(quota_descriptor["shares"])
    return quota

def new_vnfd_v3(mydb, tenant_id, vnf_descriptor):
    """
    Parses an OSM IM vnfd_catalog and insert at DB
    :param mydb:
    :param tenant_id:
    :param vnf_descriptor:
    :return: The list of cretated vnf ids
    """
    try:
        myvnfd = vnfd_catalog.vnfd()
        try:
            pybindJSONDecoder.load_ietf_json(vnf_descriptor, None, None, obj=myvnfd, path_helper=True,
                                             skip_unknown=True)
        except Exception as e:
            raise NfvoException("Error. Invalid VNF descriptor format " + str(e), httperrors.Bad_Request)
        db_vnfs = []
        db_nets = []
        db_vms = []
        db_vms_index = 0
        db_interfaces = []
        db_images = []
        db_flavors = []
        db_ip_profiles_index = 0
        db_ip_profiles = []
        uuid_list = []
        vnfd_uuid_list = []
        vnfd_catalog_descriptor = vnf_descriptor.get("vnfd:vnfd-catalog")
        if not vnfd_catalog_descriptor:
            vnfd_catalog_descriptor = vnf_descriptor.get("vnfd-catalog")
        vnfd_descriptor_list = vnfd_catalog_descriptor.get("vnfd")
        if not vnfd_descriptor_list:
            vnfd_descriptor_list = vnfd_catalog_descriptor.get("vnfd:vnfd")
        for vnfd_yang in myvnfd.vnfd_catalog.vnfd.values():
            vnfd = vnfd_yang.get()

            # table vnf
            vnf_uuid = str(uuid4())
            uuid_list.append(vnf_uuid)
            vnfd_uuid_list.append(vnf_uuid)
            vnfd_id = get_str(vnfd, "id", 255)
            db_vnf = {
                "uuid": vnf_uuid,
                "osm_id": vnfd_id,
                "name": get_str(vnfd, "name", 255),
                "description": get_str(vnfd, "description", 255),
                "tenant_id": tenant_id,
                "vendor": get_str(vnfd, "vendor", 255),
                "short_name": get_str(vnfd, "short-name", 255),
                "descriptor": str(vnf_descriptor)[:60000]
            }

            for vnfd_descriptor in vnfd_descriptor_list:
                if vnfd_descriptor["id"] == str(vnfd["id"]):
                    break

            # table ip_profiles (ip-profiles)
            ip_profile_name2db_table_index = {}
            for ip_profile in vnfd.get("ip-profiles").values():
                db_ip_profile = {
                    "ip_version": str(ip_profile["ip-profile-params"].get("ip-version", "ipv4")),
                    "subnet_address": str(ip_profile["ip-profile-params"].get("subnet-address")),
                    "gateway_address": str(ip_profile["ip-profile-params"].get("gateway-address")),
                    "dhcp_enabled": str(ip_profile["ip-profile-params"]["dhcp-params"].get("enabled", True)),
                    "dhcp_start_address": str(ip_profile["ip-profile-params"]["dhcp-params"].get("start-address")),
                    "dhcp_count": str(ip_profile["ip-profile-params"]["dhcp-params"].get("count")),
                }
                dns_list = []
                for dns in ip_profile["ip-profile-params"]["dns-server"].values():
                    dns_list.append(str(dns.get("address")))
                db_ip_profile["dns_address"] = ";".join(dns_list)
                if ip_profile["ip-profile-params"].get('security-group'):
                    db_ip_profile["security_group"] = ip_profile["ip-profile-params"]['security-group']
                ip_profile_name2db_table_index[str(ip_profile["name"])] = db_ip_profiles_index
                db_ip_profiles_index += 1
                db_ip_profiles.append(db_ip_profile)

            # table nets (internal-vld)
            net_id2uuid = {}  # for mapping interface with network
            net_id2index = {}  # for mapping interface with network
            for vld in vnfd.get("internal-vld").values():
                net_uuid = str(uuid4())
                uuid_list.append(net_uuid)
                db_net = {
                    "name": get_str(vld, "name", 255),
                    "vnf_id": vnf_uuid,
                    "uuid": net_uuid,
                    "description": get_str(vld, "description", 255),
                    "osm_id": get_str(vld, "id", 255),
                    "type": "bridge",   # TODO adjust depending on connection point type
                }
                net_id2uuid[vld.get("id")] = net_uuid
                net_id2index[vld.get("id")] = len(db_nets)
                db_nets.append(db_net)
                # ip-profile, link db_ip_profile with db_sce_net
                if vld.get("ip-profile-ref"):
                    ip_profile_name = vld.get("ip-profile-ref")
                    if ip_profile_name not in ip_profile_name2db_table_index:
                        raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{}]':'vld[{}]':'ip-profile-ref':"
                                            "'{}'. Reference to a non-existing 'ip_profiles'".format(
                                                str(vnfd["id"]), str(vld["id"]), str(vld["ip-profile-ref"])),
                                            httperrors.Bad_Request)
                    db_ip_profiles[ip_profile_name2db_table_index[ip_profile_name]]["net_id"] = net_uuid
                else:  #check no ip-address has been defined
                    for icp in vld.get("internal-connection-point").values():
                        if icp.get("ip-address"):
                            raise NfvoException("Error at 'vnfd[{}]':'vld[{}]':'internal-connection-point[{}]' "
                                            "contains an ip-address but no ip-profile has been defined at VLD".format(
                                                str(vnfd["id"]), str(vld["id"]), str(icp["id"])),
                                            httperrors.Bad_Request)

            # connection points vaiable declaration
            cp_name2iface_uuid = {}
            cp_name2vdu_id = {}
            cp_name2vm_uuid = {}
            cp_name2db_interface = {}
            vdu_id2cp_name = {}  # stored only when one external connection point is presented at this VDU

            # table vms (vdus)
            vdu_id2uuid = {}
            vdu_id2db_table_index = {}
            mgmt_access = {}
            for vdu in vnfd.get("vdu").values():

                for vdu_descriptor in vnfd_descriptor["vdu"]:
                    if vdu_descriptor["id"] == str(vdu["id"]):
                        break
                vm_uuid = str(uuid4())
                uuid_list.append(vm_uuid)
                vdu_id = get_str(vdu, "id", 255)
                db_vm = {
                    "uuid": vm_uuid,
                    "osm_id": vdu_id,
                    "name": get_str(vdu, "name", 255),
                    "description": get_str(vdu, "description", 255),
                    "pdu_type": get_str(vdu, "pdu-type", 255),
                    "vnf_id": vnf_uuid,
                }
                vdu_id2uuid[db_vm["osm_id"]] = vm_uuid
                vdu_id2db_table_index[db_vm["osm_id"]] = db_vms_index
                if vdu.get("count"):
                    db_vm["count"] = int(vdu["count"])

                # table image
                image_present = False
                if vdu.get("image"):
                    image_present = True
                    db_image = {}
                    image_uuid = _lookfor_or_create_image(db_image, mydb, vdu)
                    if not image_uuid:
                        image_uuid = db_image["uuid"]
                        db_images.append(db_image)
                    db_vm["image_id"] = image_uuid
                if vdu.get("alternative-images"):
                    vm_alternative_images = []
                    for alt_image in vdu.get("alternative-images").values():
                        db_image = {}
                        image_uuid = _lookfor_or_create_image(db_image, mydb, alt_image)
                        if not image_uuid:
                            image_uuid = db_image["uuid"]
                            db_images.append(db_image)
                        vm_alternative_images.append({
                            "image_id": image_uuid,
                            "vim_type": str(alt_image["vim-type"]),
                            # "universal_name": str(alt_image["image"]),
                            # "checksum": str(alt_image["image-checksum"]) if alt_image.get("image-checksum") else None
                        })

                    db_vm["image_list"] = yaml.safe_dump(vm_alternative_images, default_flow_style=True, width=256)

                # volumes
                devices = []
                if vdu.get("volumes"):
                    for volume_key in vdu["volumes"]:
                        volume = vdu["volumes"][volume_key]
                        if not image_present:
                            # Convert the first volume to vnfc.image
                            image_present = True
                            db_image = {}
                            image_uuid = _lookfor_or_create_image(db_image, mydb, volume)
                            if not image_uuid:
                                image_uuid = db_image["uuid"]
                                db_images.append(db_image)
                            db_vm["image_id"] = image_uuid
                        else:
                            # Add Openmano devices
                            device = {"name": str(volume.get("name"))}
                            device["type"] = str(volume.get("device-type"))
                            if volume.get("size"):
                                device["size"] = int(volume["size"])
                            if volume.get("image"):
                                device["image name"] = str(volume["image"])
                                if volume.get("image-checksum"):
                                    device["image checksum"] = str(volume["image-checksum"])

                            devices.append(device)

                if not db_vm.get("image_id"):
                    if not db_vm["pdu_type"]:
                        raise NfvoException("Not defined image for VDU")
                    # create a fake image

                # cloud-init
                boot_data = {}
                if vdu.get("cloud-init"):
                    boot_data["user-data"] = str(vdu["cloud-init"])
                elif vdu.get("cloud-init-file"):
                    # TODO Where this file content is present???
                    # boot_data["user-data"] = vnfd_yang.files[vdu["cloud-init-file"]]
                    boot_data["user-data"] = str(vdu["cloud-init-file"])

                if vdu.get("supplemental-boot-data"):
                    if vdu["supplemental-boot-data"].get('boot-data-drive'):
                            boot_data['boot-data-drive'] = True
                    if vdu["supplemental-boot-data"].get('config-file'):
                        om_cfgfile_list = list()
                        for custom_config_file in vdu["supplemental-boot-data"]['config-file'].values():
                            # TODO Where this file content is present???
                            cfg_source = str(custom_config_file["source"])
                            om_cfgfile_list.append({"dest": custom_config_file["dest"],
                                                    "content": cfg_source})
                        boot_data['config-files'] = om_cfgfile_list
                if boot_data:
                    db_vm["boot_data"] = yaml.safe_dump(boot_data, default_flow_style=True, width=256)

                db_vms.append(db_vm)
                db_vms_index += 1

                # table interfaces (internal/external interfaces)
                flavor_epa_interfaces = []
                # for iface in chain(vdu.get("internal-interface").values(), vdu.get("external-interface").values()):
                for iface in vdu.get("interface").values():
                    flavor_epa_interface = {}
                    iface_uuid = str(uuid4())
                    uuid_list.append(iface_uuid)
                    db_interface = {
                        "uuid": iface_uuid,
                        "internal_name": get_str(iface, "name", 255),
                        "vm_id": vm_uuid,
                    }
                    flavor_epa_interface["name"] = db_interface["internal_name"]
                    if iface.get("virtual-interface").get("vpci"):
                        db_interface["vpci"] = get_str(iface.get("virtual-interface"), "vpci", 12)
                        flavor_epa_interface["vpci"] = db_interface["vpci"]

                    if iface.get("virtual-interface").get("bandwidth"):
                        bps = int(iface.get("virtual-interface").get("bandwidth"))
                        db_interface["bw"] = int(math.ceil(bps / 1000000.0))
                        flavor_epa_interface["bandwidth"] = "{} Mbps".format(db_interface["bw"])

                    if iface.get("virtual-interface").get("type") == "OM-MGMT":
                        db_interface["type"] = "mgmt"
                    elif iface.get("virtual-interface").get("type") in ("VIRTIO", "E1000", "PARAVIRT"):
                        db_interface["type"] = "bridge"
                        db_interface["model"] = get_str(iface.get("virtual-interface"), "type", 12)
                    elif iface.get("virtual-interface").get("type") in ("SR-IOV", "PCI-PASSTHROUGH"):
                        db_interface["type"] = "data"
                        db_interface["model"] = get_str(iface.get("virtual-interface"), "type", 12)
                        flavor_epa_interface["dedicated"] = "no" if iface["virtual-interface"]["type"] == "SR-IOV" \
                            else "yes"
                        flavor_epa_interfaces.append(flavor_epa_interface)
                    else:
                        raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{}]':'vdu[{}]':'interface':'virtual"
                                            "-interface':'type':'{}'. Interface type is not supported".format(
                                                vnfd_id, vdu_id, iface.get("virtual-interface").get("type")),
                                            httperrors.Bad_Request)

                    if iface.get("mgmt-interface"):
                        db_interface["type"] = "mgmt"

                    if iface.get("external-connection-point-ref"):
                        try:
                            cp = vnfd.get("connection-point")[iface.get("external-connection-point-ref")]
                            db_interface["external_name"] = get_str(cp, "name", 255)
                            cp_name2iface_uuid[db_interface["external_name"]] = iface_uuid
                            cp_name2vdu_id[db_interface["external_name"]] = vdu_id
                            cp_name2vm_uuid[db_interface["external_name"]] = vm_uuid
                            cp_name2db_interface[db_interface["external_name"]] = db_interface
                            for cp_descriptor in vnfd_descriptor["connection-point"]:
                                if cp_descriptor["name"] == db_interface["external_name"]:
                                    break
                            else:
                                raise KeyError()

                            if vdu_id in vdu_id2cp_name:
                                vdu_id2cp_name[vdu_id] = None  # more than two connection point for this VDU
                            else:
                                vdu_id2cp_name[vdu_id] = db_interface["external_name"]

                            # port security
                            if str(cp_descriptor.get("port-security-enabled")).lower() == "false":
                                db_interface["port_security"] = 0
                            elif str(cp_descriptor.get("port-security-enabled")).lower() == "true":
                                db_interface["port_security"] = 1
                        except KeyError:
                            raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'vdu[{vdu}]':"
                                                "'interface[{iface}]':'vnfd-connection-point-ref':'{cp}' is not present"
                                                " at connection-point".format(
                                                    vnf=vnfd_id, vdu=vdu_id, iface=iface["name"],
                                                    cp=iface.get("vnfd-connection-point-ref")),
                                                httperrors.Bad_Request)
                    elif iface.get("internal-connection-point-ref"):
                        try:
                            for icp_descriptor in vdu_descriptor["internal-connection-point"]:
                                if icp_descriptor["id"] == str(iface.get("internal-connection-point-ref")):
                                    break
                            else:
                                raise KeyError("does not exist at vdu:internal-connection-point")
                            icp = None
                            icp_vld = None
                            for vld in vnfd.get("internal-vld").values():
                                for cp in vld.get("internal-connection-point").values():
                                    if cp.get("id-ref") == iface.get("internal-connection-point-ref"):
                                        if icp:
                                            raise KeyError("is referenced by more than one 'internal-vld'")
                                        icp = cp
                                        icp_vld = vld
                            if not icp:
                                raise KeyError("is not referenced by any 'internal-vld'")

                            # set network type as data
                            if iface.get("virtual-interface") and iface["virtual-interface"].get("type") in \
                                    ("SR-IOV", "PCI-PASSTHROUGH"):
                                db_nets[net_id2index[icp_vld.get("id")]]["type"] = "data"
                            db_interface["net_id"] = net_id2uuid[icp_vld.get("id")]
                            if str(icp_descriptor.get("port-security-enabled")).lower() == "false":
                                db_interface["port_security"] = 0
                            elif str(icp_descriptor.get("port-security-enabled")).lower() == "true":
                                db_interface["port_security"] = 1
                            if icp.get("ip-address"):
                                if not icp_vld.get("ip-profile-ref"):
                                    raise NfvoException
                                db_interface["ip_address"] = str(icp.get("ip-address"))
                        except KeyError as e:
                            raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'vdu[{vdu}]':"
                                                "'interface[{iface}]':'internal-connection-point-ref':'{cp}'"
                                                " {msg}".format(
                                                    vnf=vnfd_id, vdu=vdu_id, iface=iface["name"],
                                                    cp=iface.get("internal-connection-point-ref"), msg=str(e)),
                                                httperrors.Bad_Request)
                    if iface.get("position"):
                        db_interface["created_at"] = int(iface.get("position")) * 50
                    if iface.get("mac-address"):
                        db_interface["mac"] = str(iface.get("mac-address"))
                    db_interfaces.append(db_interface)

                # table flavors
                db_flavor = {
                    "name": get_str(vdu, "name", 250) + "-flv",
                    "vcpus": int(vdu["vm-flavor"].get("vcpu-count", 1)),
                    "ram": int(vdu["vm-flavor"].get("memory-mb", 1)),
                    "disk": int(vdu["vm-flavor"].get("storage-gb", 0)),
                }
                # TODO revise the case of several numa-node-policy node
                extended = {}
                numa = {}
                if devices:
                    extended["devices"] = devices
                if flavor_epa_interfaces:
                    numa["interfaces"] = flavor_epa_interfaces
                if vdu.get("guest-epa"):   # TODO or dedicated_int:
                    epa_vcpu_set = False
                    if vdu["guest-epa"].get("numa-node-policy"):  # TODO or dedicated_int:
                        numa_node_policy = vdu["guest-epa"].get("numa-node-policy")
                        if numa_node_policy.get("node"):
                            numa_node = next(iter(numa_node_policy["node"].values()))
                            if numa_node.get("num-cores"):
                                numa["cores"] = numa_node["num-cores"]
                                epa_vcpu_set = True
                            if numa_node.get("paired-threads"):
                                if numa_node["paired-threads"].get("num-paired-threads"):
                                    numa["paired-threads"] = int(numa_node["paired-threads"]["num-paired-threads"])
                                    epa_vcpu_set = True
                                if len(numa_node["paired-threads"].get("paired-thread-ids")):
                                    numa["paired-threads-id"] = []
                                    for pair in numa_node["paired-threads"]["paired-thread-ids"].values():
                                        numa["paired-threads-id"].append(
                                            (str(pair["thread-a"]), str(pair["thread-b"]))
                                        )
                            if numa_node.get("num-threads"):
                                numa["threads"] = int(numa_node["num-threads"])
                                epa_vcpu_set = True
                            if numa_node.get("memory-mb"):
                                numa["memory"] = max(int(numa_node["memory-mb"] / 1024), 1)
                    if vdu["guest-epa"].get("mempage-size"):
                        if vdu["guest-epa"]["mempage-size"] != "SMALL":
                            numa["memory"] = max(int(db_flavor["ram"] / 1024), 1)
                    if vdu["guest-epa"].get("cpu-pinning-policy") and not epa_vcpu_set:
                        if vdu["guest-epa"]["cpu-pinning-policy"] == "DEDICATED":
                            if vdu["guest-epa"].get("cpu-thread-pinning-policy") and \
                                            vdu["guest-epa"]["cpu-thread-pinning-policy"] != "PREFER":
                                numa["cores"] = max(db_flavor["vcpus"], 1)
                            else:
                                numa["threads"] = max(db_flavor["vcpus"], 1)
                            epa_vcpu_set = True
                    if vdu["guest-epa"].get("cpu-quota") and not epa_vcpu_set:
                        cpuquota = get_resource_allocation_params(vdu["guest-epa"].get("cpu-quota"))
                        if cpuquota:
                            extended["cpu-quota"] = cpuquota
                    if vdu["guest-epa"].get("mem-quota"):
                        vduquota = get_resource_allocation_params(vdu["guest-epa"].get("mem-quota"))
                        if vduquota:
                            extended["mem-quota"] = vduquota
                    if vdu["guest-epa"].get("disk-io-quota"):
                        diskioquota = get_resource_allocation_params(vdu["guest-epa"].get("disk-io-quota"))
                        if diskioquota:
                            extended["disk-io-quota"] = diskioquota
                    if vdu["guest-epa"].get("vif-quota"):
                        vifquota = get_resource_allocation_params(vdu["guest-epa"].get("vif-quota"))
                        if vifquota:
                            extended["vif-quota"] = vifquota
                if numa:
                    extended["numas"] = [numa]
                if extended:
                    extended_text = yaml.safe_dump(extended, default_flow_style=True, width=256)
                    db_flavor["extended"] = extended_text
                # look if flavor exist
                temp_flavor_dict = {'disk': db_flavor.get('disk', 0),
                                    'ram': db_flavor.get('ram'),
                                    'vcpus': db_flavor.get('vcpus'),
                                    'extended': db_flavor.get('extended')
                                    }
                existing_flavors = mydb.get_rows(FROM="flavors", WHERE=temp_flavor_dict)
                if existing_flavors:
                    flavor_uuid = existing_flavors[0]["uuid"]
                else:
                    flavor_uuid = str(uuid4())
                    uuid_list.append(flavor_uuid)
                    db_flavor["uuid"] = flavor_uuid
                    db_flavors.append(db_flavor)
                db_vm["flavor_id"] = flavor_uuid

            # VNF affinity and antiaffinity
            for pg in vnfd.get("placement-groups").values():
                pg_name = get_str(pg, "name", 255)
                for vdu in pg.get("member-vdus").values():
                    vdu_id = get_str(vdu, "member-vdu-ref", 255)
                    if vdu_id not in vdu_id2db_table_index:
                        raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'placement-groups[{pg}]':"
                                            "'member-vdus':'{vdu}'. Reference to a non-existing vdu".format(
                                                vnf=vnfd_id, pg=pg_name, vdu=vdu_id),
                                            httperrors.Bad_Request)
                    db_vms[vdu_id2db_table_index[vdu_id]]["availability_zone"] = pg_name
                    # TODO consider the case of isolation and not colocation
                    # if pg.get("strategy") == "ISOLATION":

            # VNF mgmt configuration
            if vnfd["mgmt-interface"].get("vdu-id"):
                mgmt_vdu_id = get_str(vnfd["mgmt-interface"], "vdu-id", 255)
                if mgmt_vdu_id not in vdu_id2uuid:
                    raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'mgmt-interface':'vdu-id':"
                                        "'{vdu}'. Reference to a non-existing vdu".format(
                                            vnf=vnfd_id, vdu=mgmt_vdu_id),
                                        httperrors.Bad_Request)
                mgmt_access["vm_id"] = vdu_id2uuid[mgmt_vdu_id]
                mgmt_access["vdu-id"] = mgmt_vdu_id
                # if only one cp is defined by this VDU, mark this interface as of type "mgmt"
                if vdu_id2cp_name.get(mgmt_vdu_id):
                    if cp_name2db_interface[vdu_id2cp_name[mgmt_vdu_id]]:
                        cp_name2db_interface[vdu_id2cp_name[mgmt_vdu_id]]["type"] = "mgmt"

            if vnfd["mgmt-interface"].get("ip-address"):
                mgmt_access["ip-address"] = str(vnfd["mgmt-interface"].get("ip-address"))
            if vnfd["mgmt-interface"].get("cp") and vnfd.get("vdu"):
                if vnfd["mgmt-interface"]["cp"] not in cp_name2iface_uuid:
                    raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'mgmt-interface':'cp'['{cp}']. "
                                        "Reference to a non-existing connection-point".format(
                                            vnf=vnfd_id, cp=vnfd["mgmt-interface"]["cp"]),
                                        httperrors.Bad_Request)
                mgmt_access["vm_id"] = cp_name2vm_uuid[vnfd["mgmt-interface"]["cp"]]
                mgmt_access["interface_id"] = cp_name2iface_uuid[vnfd["mgmt-interface"]["cp"]]
                mgmt_access["vdu-id"] = cp_name2vdu_id[vnfd["mgmt-interface"]["cp"]]
                # mark this interface as of type mgmt
                if cp_name2db_interface[vnfd["mgmt-interface"]["cp"]]:
                    cp_name2db_interface[vnfd["mgmt-interface"]["cp"]]["type"] = "mgmt"

            default_user = get_str(vnfd.get("vnf-configuration", {}).get("config-access", {}).get("ssh-access", {}),
                                    "default-user", 64)
            if default_user:
                mgmt_access["default_user"] = default_user

            required = get_str(vnfd.get("vnf-configuration", {}).get("config-access", {}).get("ssh-access", {}),
                                   "required", 6)
            if required:
                mgmt_access["required"] = required

            password_ = get_str(vnfd.get("vnf-configuration", {}).get("config-access", {}),
                                   "password", 64)
            if password_:
                mgmt_access["password"] = password_

            if mgmt_access:
                db_vnf["mgmt_access"] = yaml.safe_dump(mgmt_access, default_flow_style=True, width=256)

            db_vnfs.append(db_vnf)
        db_tables=[
            {"vnfs": db_vnfs},
            {"nets": db_nets},
            {"images": db_images},
            {"flavors": db_flavors},
            {"ip_profiles": db_ip_profiles},
            {"vms": db_vms},
            {"interfaces": db_interfaces},
        ]

        logger.debug("create_vnf Deployment done vnfDict: %s",
                    yaml.safe_dump(db_tables, indent=4, default_flow_style=False) )
        mydb.new_rows(db_tables, uuid_list)
        return vnfd_uuid_list
    except NfvoException:
        raise
    except Exception as e:
        logger.error("Exception {}".format(e))
        raise  # NfvoException("Exception {}".format(e), httperrors.Bad_Request)


@deprecated("Use new_vnfd_v3")
def new_vnf(mydb, tenant_id, vnf_descriptor):
    global global_config

    # Step 1. Check the VNF descriptor
    check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1)
    # Step 2. Check tenant exist
    vims = {}
    if tenant_id != "any":
        check_tenant(mydb, tenant_id)
        if "tenant_id" in vnf_descriptor["vnf"]:
            if vnf_descriptor["vnf"]["tenant_id"] != tenant_id:
                raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(vnf_descriptor["vnf"]["tenant_id"], tenant_id),
                                    httperrors.Unauthorized)
        else:
            vnf_descriptor['vnf']['tenant_id'] = tenant_id
        # Step 3. Get the URL of the VIM from the nfvo_tenant and the datacenter
        if global_config["auto_push_VNF_to_VIMs"]:
            vims = get_vim(mydb, tenant_id, ignore_errors=True)

    # Step 4. Review the descriptor and add missing  fields
    #print vnf_descriptor
    #logger.debug("Refactoring VNF descriptor with fields: description, public (default: true)")
    vnf_name = vnf_descriptor['vnf']['name']
    vnf_descriptor['vnf']['description'] = vnf_descriptor['vnf'].get("description", vnf_name)
    if "physical" in vnf_descriptor['vnf']:
        del vnf_descriptor['vnf']['physical']
    #print vnf_descriptor

    # Step 6. For each VNFC in the descriptor, flavors and images are created in the VIM
    logger.debug('BEGIN creation of VNF "%s"' % vnf_name)
    logger.debug("VNF %s: consisting of %d VNFC(s)" % (vnf_name,len(vnf_descriptor['vnf']['VNFC'])))

    #For each VNFC, we add it to the VNFCDict and we  create a flavor.
    VNFCDict = {}     # Dictionary, key: VNFC name, value: dict with the relevant information to create the VNF and VMs in the MANO database
    rollback_list = []    # It will contain the new images created in mano. It is used for rollback
    try:
        logger.debug("Creating additional disk images and new flavors in the VIM for each VNFC")
        for vnfc in vnf_descriptor['vnf']['VNFC']:
            VNFCitem={}
            VNFCitem["name"] = vnfc['name']
            VNFCitem["availability_zone"] = vnfc.get('availability_zone')
            VNFCitem["description"] = vnfc.get("description", 'VM {} of the VNF {}'.format(vnfc['name'],vnf_name))

            #print "Flavor name: %s. Description: %s" % (VNFCitem["name"]+"-flv", VNFCitem["description"])

            myflavorDict = {}
            myflavorDict["name"] = vnfc['name']+"-flv"   #Maybe we could rename the flavor by using the field "image name" if exists
            myflavorDict["description"] = VNFCitem["description"]
            myflavorDict["ram"] = vnfc.get("ram", 0)
            myflavorDict["vcpus"] = vnfc.get("vcpus", 0)
            myflavorDict["disk"] = vnfc.get("disk", 0)
            myflavorDict["extended"] = {}

            devices = vnfc.get("devices")
            if devices != None:
                myflavorDict["extended"]["devices"] = devices

            # TODO:
            # Mapping from processor models to rankings should be available somehow in the NFVO. They could be taken from VIM or directly from a new database table
            # Another option is that the processor in the VNF descriptor specifies directly the ranking of the host

            # Previous code has been commented
            #if vnfc['processor']['model'] == "Intel(R) Xeon(R) CPU E5-4620 0 @ 2.20GHz" :
            #    myflavorDict["flavor"]['extended']['processor_ranking'] = 200
            #elif vnfc['processor']['model'] == "Intel(R) Xeon(R) CPU E5-2697 v2 @ 2.70GHz" :
            #    myflavorDict["flavor"]['extended']['processor_ranking'] = 300
            #else:
            #    result2, message = rollback(myvim, myvimURL, myvim_tenant, flavorList, imageList)
            #    if result2:
            #        print "Error creating flavor: unknown processor model. Rollback successful."
            #        return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback successful."
            #    else:
            #        return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback fail: you need to access VIM and delete the following %s" % message
            myflavorDict['extended']['processor_ranking'] = 100  #Hardcoded value, while we decide when the mapping is done

            if 'numas' in vnfc and len(vnfc['numas'])>0:
                myflavorDict['extended']['numas'] = vnfc['numas']

            #print myflavorDict

            # Step 6.2 New flavors are created in the VIM
            flavor_id = create_or_use_flavor(mydb, vims, myflavorDict, rollback_list)

            #print "Flavor id for VNFC %s: %s" % (vnfc['name'],flavor_id)
            VNFCitem["flavor_id"] = flavor_id
            VNFCDict[vnfc['name']] = VNFCitem

        logger.debug("Creating new images in the VIM for each VNFC")
        # Step 6.3 New images are created in the VIM
        #For each VNFC, we must create the appropriate image.
        #This "for" loop might be integrated with the previous one
        #In case this integration is made, the VNFCDict might become a VNFClist.
        for vnfc in vnf_descriptor['vnf']['VNFC']:
            #print "Image name: %s. Description: %s" % (vnfc['name']+"-img", VNFCDict[vnfc['name']]['description'])
            image_dict={}
            image_dict['name']=vnfc.get('image name',vnf_name+"-"+vnfc['name']+"-img")
            image_dict['universal_name']=vnfc.get('image name')
            image_dict['description']=vnfc.get('image name', VNFCDict[vnfc['name']]['description'])
            image_dict['location']=vnfc.get('VNFC image')
            #image_dict['new_location']=vnfc.get('image location')
            image_dict['checksum']=vnfc.get('image checksum')
            image_metadata_dict = vnfc.get('image metadata', None)
            image_metadata_str = None
            if image_metadata_dict is not None:
                image_metadata_str = yaml.safe_dump(image_metadata_dict,default_flow_style=True,width=256)
            image_dict['metadata']=image_metadata_str
            #print "create_or_use_image", mydb, vims, image_dict, rollback_list
            image_id = create_or_use_image(mydb, vims, image_dict, rollback_list)
            #print "Image id for VNFC %s: %s" % (vnfc['name'],image_id)
            VNFCDict[vnfc['name']]["image_id"] = image_id
            VNFCDict[vnfc['name']]["image_path"] = vnfc.get('VNFC image')
            VNFCDict[vnfc['name']]["count"] = vnfc.get('count', 1)
            if vnfc.get("boot-data"):
                VNFCDict[vnfc['name']]["boot_data"] = yaml.safe_dump(vnfc["boot-data"], default_flow_style=True, width=256)


        # Step 7. Storing the VNF descriptor in the repository
        if "descriptor" not in vnf_descriptor["vnf"]:
            vnf_descriptor["vnf"]["descriptor"] = yaml.safe_dump(vnf_descriptor, indent=4, explicit_start=True, default_flow_style=False)

        # Step 8. Adding the VNF to the NFVO DB
        vnf_id = mydb.new_vnf_as_a_whole(tenant_id,vnf_name,vnf_descriptor,VNFCDict)
        return vnf_id
    except (db_base_Exception, vimconn.VimConnException, KeyError) as e:
        _, message = rollback(mydb, vims, rollback_list)
        if isinstance(e, db_base_Exception):
            error_text = "Exception at database"
        elif isinstance(e, KeyError):
            error_text = "KeyError exception "
            e.http_code = httperrors.Internal_Server_Error
        else:
            error_text = "Exception at VIM"
        error_text += " {} {}. {}".format(type(e).__name__, str(e), message)
        #logger.error("start_scenario %s", error_text)
        raise NfvoException(error_text, e.http_code)


@deprecated("Use new_vnfd_v3")
def new_vnf_v02(mydb, tenant_id, vnf_descriptor):
    global global_config

    # Step 1. Check the VNF descriptor
    check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=2)
    # Step 2. Check tenant exist
    vims = {}
    if tenant_id != "any":
        check_tenant(mydb, tenant_id)
        if "tenant_id" in vnf_descriptor["vnf"]:
            if vnf_descriptor["vnf"]["tenant_id"] != tenant_id:
                raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(vnf_descriptor["vnf"]["tenant_id"], tenant_id),
                                    httperrors.Unauthorized)
        else:
            vnf_descriptor['vnf']['tenant_id'] = tenant_id
        # Step 3. Get the URL of the VIM from the nfvo_tenant and the datacenter
        if global_config["auto_push_VNF_to_VIMs"]:
            vims = get_vim(mydb, tenant_id, ignore_errors=True)

    # Step 4. Review the descriptor and add missing  fields
    #print vnf_descriptor
    #logger.debug("Refactoring VNF descriptor with fields: description, public (default: true)")
    vnf_name = vnf_descriptor['vnf']['name']
    vnf_descriptor['vnf']['description'] = vnf_descriptor['vnf'].get("description", vnf_name)
    if "physical" in vnf_descriptor['vnf']:
        del vnf_descriptor['vnf']['physical']
    #print vnf_descriptor

    # Step 6. For each VNFC in the descriptor, flavors and images are created in the VIM
    logger.debug('BEGIN creation of VNF "%s"' % vnf_name)
    logger.debug("VNF %s: consisting of %d VNFC(s)" % (vnf_name,len(vnf_descriptor['vnf']['VNFC'])))

    #For each VNFC, we add it to the VNFCDict and we  create a flavor.
    VNFCDict = {}     # Dictionary, key: VNFC name, value: dict with the relevant information to create the VNF and VMs in the MANO database
    rollback_list = []    # It will contain the new images created in mano. It is used for rollback
    try:
        logger.debug("Creating additional disk images and new flavors in the VIM for each VNFC")
        for vnfc in vnf_descriptor['vnf']['VNFC']:
            VNFCitem={}
            VNFCitem["name"] = vnfc['name']
            VNFCitem["description"] = vnfc.get("description", 'VM {} of the VNF {}'.format(vnfc['name'],vnf_name))

            #print "Flavor name: %s. Description: %s" % (VNFCitem["name"]+"-flv", VNFCitem["description"])

            myflavorDict = {}
            myflavorDict["name"] = vnfc['name']+"-flv"   #Maybe we could rename the flavor by using the field "image name" if exists
            myflavorDict["description"] = VNFCitem["description"]
            myflavorDict["ram"] = vnfc.get("ram", 0)
            myflavorDict["vcpus"] = vnfc.get("vcpus", 0)
            myflavorDict["disk"] = vnfc.get("disk", 0)
            myflavorDict["extended"] = {}

            devices = vnfc.get("devices")
            if devices != None:
                myflavorDict["extended"]["devices"] = devices

            # TODO:
            # Mapping from processor models to rankings should be available somehow in the NFVO. They could be taken from VIM or directly from a new database table
            # Another option is that the processor in the VNF descriptor specifies directly the ranking of the host

            # Previous code has been commented
            #if vnfc['processor']['model'] == "Intel(R) Xeon(R) CPU E5-4620 0 @ 2.20GHz" :
            #    myflavorDict["flavor"]['extended']['processor_ranking'] = 200
            #elif vnfc['processor']['model'] == "Intel(R) Xeon(R) CPU E5-2697 v2 @ 2.70GHz" :
            #    myflavorDict["flavor"]['extended']['processor_ranking'] = 300
            #else:
            #    result2, message = rollback(myvim, myvimURL, myvim_tenant, flavorList, imageList)
            #    if result2:
            #        print "Error creating flavor: unknown processor model. Rollback successful."
            #        return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback successful."
            #    else:
            #        return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback fail: you need to access VIM and delete the following %s" % message
            myflavorDict['extended']['processor_ranking'] = 100  #Hardcoded value, while we decide when the mapping is done

            if 'numas' in vnfc and len(vnfc['numas'])>0:
                myflavorDict['extended']['numas'] = vnfc['numas']

            #print myflavorDict

            # Step 6.2 New flavors are created in the VIM
            flavor_id = create_or_use_flavor(mydb, vims, myflavorDict, rollback_list)

            #print "Flavor id for VNFC %s: %s" % (vnfc['name'],flavor_id)
            VNFCitem["flavor_id"] = flavor_id
            VNFCDict[vnfc['name']] = VNFCitem

        logger.debug("Creating new images in the VIM for each VNFC")
        # Step 6.3 New images are created in the VIM
        #For each VNFC, we must create the appropriate image.
        #This "for" loop might be integrated with the previous one
        #In case this integration is made, the VNFCDict might become a VNFClist.
        for vnfc in vnf_descriptor['vnf']['VNFC']:
            #print "Image name: %s. Description: %s" % (vnfc['name']+"-img", VNFCDict[vnfc['name']]['description'])
            image_dict={}
            image_dict['name']=vnfc.get('image name',vnf_name+"-"+vnfc['name']+"-img")
            image_dict['universal_name']=vnfc.get('image name')
            image_dict['description']=vnfc.get('image name', VNFCDict[vnfc['name']]['description'])
            image_dict['location']=vnfc.get('VNFC image')
            #image_dict['new_location']=vnfc.get('image location')
            image_dict['checksum']=vnfc.get('image checksum')
            image_metadata_dict = vnfc.get('image metadata', None)
            image_metadata_str = None
            if image_metadata_dict is not None:
                image_metadata_str = yaml.safe_dump(image_metadata_dict,default_flow_style=True,width=256)
            image_dict['metadata']=image_metadata_str
            #print "create_or_use_image", mydb, vims, image_dict, rollback_list
            image_id = create_or_use_image(mydb, vims, image_dict, rollback_list)
            #print "Image id for VNFC %s: %s" % (vnfc['name'],image_id)
            VNFCDict[vnfc['name']]["image_id"] = image_id
            VNFCDict[vnfc['name']]["image_path"] = vnfc.get('VNFC image')
            VNFCDict[vnfc['name']]["count"] = vnfc.get('count', 1)
            if vnfc.get("boot-data"):
                VNFCDict[vnfc['name']]["boot_data"] = yaml.safe_dump(vnfc["boot-data"], default_flow_style=True, width=256)

        # Step 7. Storing the VNF descriptor in the repository
        if "descriptor" not in vnf_descriptor["vnf"]:
            vnf_descriptor["vnf"]["descriptor"] = yaml.safe_dump(vnf_descriptor, indent=4, explicit_start=True, default_flow_style=False)

        # Step 8. Adding the VNF to the NFVO DB
        vnf_id = mydb.new_vnf_as_a_whole2(tenant_id,vnf_name,vnf_descriptor,VNFCDict)
        return vnf_id
    except (db_base_Exception, vimconn.VimConnException, KeyError) as e:
        _, message = rollback(mydb, vims, rollback_list)
        if isinstance(e, db_base_Exception):
            error_text = "Exception at database"
        elif isinstance(e, KeyError):
            error_text = "KeyError exception "
            e.http_code = httperrors.Internal_Server_Error
        else:
            error_text = "Exception at VIM"
        error_text += " {} {}. {}".format(type(e).__name__, str(e), message)
        #logger.error("start_scenario %s", error_text)
        raise NfvoException(error_text, e.http_code)


def get_vnf_id(mydb, tenant_id, vnf_id):
    #check valid tenant_id
    check_tenant(mydb, tenant_id)
    #obtain data
    where_or = {}
    if tenant_id != "any":
        where_or["tenant_id"] = tenant_id
        where_or["public"] = True
    vnf = mydb.get_table_by_uuid_name('vnfs', vnf_id, "VNF", WHERE_OR=where_or, WHERE_AND_OR="AND")

    vnf_id = vnf["uuid"]
    filter_keys = ('uuid', 'name', 'description', 'public', "tenant_id", "osm_id", "created_at")
    filtered_content = dict( (k,v) for k,v in vnf.items() if k in filter_keys )
    #change_keys_http2db(filtered_content, http2db_vnf, reverse=True)
    data={'vnf' : filtered_content}
    #GET VM
    content = mydb.get_rows(FROM='vnfs join vms on vnfs.uuid=vms.vnf_id',
            SELECT=('vms.uuid as uuid', 'vms.osm_id as osm_id', 'vms.name as name', 'vms.description as description',
                    'boot_data'),
            WHERE={'vnfs.uuid': vnf_id} )
    if len(content) != 0:
        #raise NfvoException("vnf '{}' not found".format(vnf_id), httperrors.Not_Found)
    # change boot_data into boot-data
        for vm in content:
            if vm.get("boot_data"):
                vm["boot-data"] = yaml.safe_load(vm["boot_data"])
                del vm["boot_data"]

        data['vnf']['VNFC'] = content
    #TODO: GET all the information from a VNFC and include it in the output.

    #GET NET
    content = mydb.get_rows(FROM='vnfs join nets on vnfs.uuid=nets.vnf_id',
                                    SELECT=('nets.uuid as uuid','nets.name as name','nets.description as description', 'nets.type as type', 'nets.multipoint as multipoint'),
                                    WHERE={'vnfs.uuid': vnf_id} )
    data['vnf']['nets'] = content

    #GET ip-profile for each net
    for net in data['vnf']['nets']:
        ipprofiles = mydb.get_rows(FROM='ip_profiles',
                                   SELECT=('ip_version','subnet_address','gateway_address','dns_address','dhcp_enabled','dhcp_start_address','dhcp_count'),
                                   WHERE={'net_id': net["uuid"]} )
        if len(ipprofiles)==1:
            net["ip_profile"] = ipprofiles[0]
        elif len(ipprofiles)>1:
            raise NfvoException("More than one ip-profile found with this criteria: net_id='{}'".format(net['uuid']), httperrors.Bad_Request)


    #TODO: For each net, GET its elements and relevant info per element (VNFC, iface, ip_address) and include them in the output.

    #GET External Interfaces
    content = mydb.get_rows(FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces on vms.uuid=interfaces.vm_id',\
                                    SELECT=('interfaces.uuid as uuid','interfaces.external_name as external_name', 'vms.name as vm_name', 'interfaces.vm_id as vm_id', \
                                            'interfaces.internal_name as internal_name', 'interfaces.type as type', 'interfaces.vpci as vpci','interfaces.bw as bw'),\
                                    WHERE={'vnfs.uuid': vnf_id, 'interfaces.external_name<>': None} )
    #print content
    data['vnf']['external-connections'] = content

    return data


def delete_vnf(mydb,tenant_id,vnf_id,datacenter=None,vim_tenant=None):
    # Check tenant exist
    if tenant_id != "any":
        check_tenant(mydb, tenant_id)
        # Get the URL of the VIM from the nfvo_tenant and the datacenter
        vims = get_vim(mydb, tenant_id, ignore_errors=True)
    else:
        vims={}

    # Checking if it is a valid uuid and, if not, getting the uuid assuming that the name was provided"
    where_or = {}
    if tenant_id != "any":
        where_or["tenant_id"] = tenant_id
        where_or["public"] = True
    vnf = mydb.get_table_by_uuid_name('vnfs', vnf_id, "VNF", WHERE_OR=where_or, WHERE_AND_OR="AND")
    vnf_id = vnf["uuid"]

    # "Getting the list of flavors and tenants of the VNF"
    flavorList = get_flavorlist(mydb, vnf_id)
    if len(flavorList)==0:
        logger.warn("delete_vnf error. No flavors found for the VNF id '%s'", vnf_id)

    imageList = get_imagelist(mydb, vnf_id)
    if len(imageList)==0:
        logger.warn( "delete_vnf error. No images found for the VNF id '%s'", vnf_id)

    deleted = mydb.delete_row_by_id('vnfs', vnf_id)
    if deleted == 0:
        raise NfvoException("vnf '{}' not found".format(vnf_id), httperrors.Not_Found)

    undeletedItems = []
    for flavor in flavorList:
        #check if flavor is used by other vnf
        try:
            c = mydb.get_rows(FROM='vms', WHERE={'flavor_id':flavor} )
            if len(c) > 0:
                logger.debug("Flavor '%s' not deleted because it is being used by another VNF", flavor)
                continue
            #flavor not used, must be deleted
            #delelte at VIM
            c = mydb.get_rows(FROM='datacenters_flavors', WHERE={'flavor_id': flavor})
            for flavor_vim in c:
                if not flavor_vim['created']:  # skip this flavor because not created by openmano
                    continue
                # look for vim
                myvim = None
                for vim in vims.values():
                    if vim["config"]["datacenter_tenant_id"] == flavor_vim["datacenter_vim_id"]:
                        myvim = vim
                        break
                if not myvim:
                    continue
                try:
                    myvim.delete_flavor(flavor_vim["vim_id"])
                except vimconn.VimConnNotFoundException:
                    logger.warn("VIM flavor %s not exist at datacenter %s", flavor_vim["vim_id"],
                                flavor_vim["datacenter_vim_id"] )
                except vimconn.VimConnException as e:
                    logger.error("Not possible to delete VIM flavor %s from datacenter %s: %s %s",
                            flavor_vim["vim_id"], flavor_vim["datacenter_vim_id"], type(e).__name__, str(e))
                    undeletedItems.append("flavor {} from VIM {}".format(flavor_vim["vim_id"],
                                                                         flavor_vim["datacenter_vim_id"]))
            # delete flavor from Database, using table flavors and with cascade foreign key also at datacenters_flavors
            mydb.delete_row_by_id('flavors', flavor)
        except db_base_Exception as e:
            logger.error("delete_vnf_error. Not possible to get flavor details and delete '%s'. %s", flavor, str(e))
            undeletedItems.append("flavor {}".format(flavor))


    for image in imageList:
        try:
            #check if image is used by other vnf
            c = mydb.get_rows(FROM='vms', WHERE=[{'image_id': image}, {'image_list LIKE ': '%' + image + '%'}])
            if len(c) > 0:
                logger.debug("Image '%s' not deleted because it is being used by another VNF", image)
                continue
            #image not used, must be deleted
            #delelte at VIM
            c = mydb.get_rows(FROM='datacenters_images', WHERE={'image_id':image})
            for image_vim in c:
                if image_vim["datacenter_vim_id"] not in vims:   # TODO change to datacenter_tenant_id
                    continue
                if image_vim['created']=='false': #skip this image because not created by openmano
                    continue
                myvim=vims[ image_vim["datacenter_id"] ]
                try:
                    myvim.delete_image(image_vim["vim_id"])
                except vimconn.VimConnNotFoundException as e:
                    logger.warn("VIM image %s not exist at datacenter %s", image_vim["vim_id"], image_vim["datacenter_id"] )
                except vimconn.VimConnException as e:
                    logger.error("Not possible to delete VIM image %s from datacenter %s: %s %s",
                            image_vim["vim_id"], image_vim["datacenter_id"], type(e).__name__, str(e))
                    undeletedItems.append("image {} from VIM {}".format(image_vim["vim_id"], image_vim["datacenter_id"] ))
            #delete image from Database, using table images and with cascade foreign key also at datacenters_images
            mydb.delete_row_by_id('images', image)
        except db_base_Exception as e:
            logger.error("delete_vnf_error. Not possible to get image details and delete '%s'. %s", image, str(e))
            undeletedItems.append("image {}".format(image))

    return vnf_id + " " + vnf["name"]
    #if undeletedItems:
    #    return "delete_vnf. Undeleted: %s" %(undeletedItems)


@deprecated("Not used")
def get_hosts_info(mydb, nfvo_tenant_id, datacenter_name=None):
    result, vims = get_vim(mydb, nfvo_tenant_id, None, datacenter_name)
    if result < 0:
        return result, vims
    elif result == 0:
        return -httperrors.Not_Found, "datacenter '{}' not found".format(datacenter_name)
    myvim = next(iter(vims.values()))
    result,servers =  myvim.get_hosts_info()
    if result < 0:
        return result, servers
    topology = {'name':myvim['name'] , 'servers': servers}
    return result, topology


def get_hosts(mydb, nfvo_tenant_id):
    vims = get_vim(mydb, nfvo_tenant_id)
    if len(vims) == 0:
        raise NfvoException("No datacenter found for tenant '{}'".format(str(nfvo_tenant_id)), httperrors.Not_Found)
    elif len(vims)>1:
        #print "nfvo.datacenter_action() error. Several datacenters found"
        raise NfvoException("More than one datacenters found, try to identify with uuid", httperrors.Conflict)
    myvim = next(iter(vims.values()))
    try:
        hosts =  myvim.get_hosts()
        logger.debug('VIM hosts response: '+ yaml.safe_dump(hosts, indent=4, default_flow_style=False))

        datacenter = {'Datacenters': [ {'name':myvim['name'],'servers':[]} ] }
        for host in hosts:
            server={'name':host['name'], 'vms':[]}
            for vm in host['instances']:
                #get internal name and model
                try:
                    c = mydb.get_rows(SELECT=('name',), FROM='instance_vms as iv join vms on iv.vm_id=vms.uuid',\
                        WHERE={'vim_vm_id':vm['id']} )
                    if len(c) == 0:
                        logger.warn("nfvo.get_hosts virtual machine at VIM '{}' not found at tidnfvo".format(vm['id']))
                        continue
                    server['vms'].append( {'name':vm['name'] , 'model':c[0]['name']} )

                except db_base_Exception as e:
                    logger.warn("nfvo.get_hosts virtual machine at VIM '{}' error {}".format(vm['id'], str(e)))
            datacenter['Datacenters'][0]['servers'].append(server)
        #return -400, "en construccion"

        #print 'datacenters '+ json.dumps(datacenter, indent=4)
        return datacenter
    except vimconn.VimConnException as e:
        raise NfvoException("Not possible to get_host_list from VIM: {}".format(str(e)), e.http_code)


@deprecated("Use new_nsd_v3")
def new_scenario(mydb, tenant_id, topo):

#    result, vims = get_vim(mydb, tenant_id)
#    if result < 0:
#        return result, vims
#1: parse input
    if tenant_id != "any":
        check_tenant(mydb, tenant_id)
        if "tenant_id" in topo:
            if topo["tenant_id"] != tenant_id:
                raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(topo["tenant_id"], tenant_id),
                                    httperrors.Unauthorized)
    else:
        tenant_id=None

#1.1: get VNFs and external_networks (other_nets).
    vnfs={}
    other_nets={}  #external_networks, bridge_networks and data_networkds
    nodes = topo['topology']['nodes']
    for k in nodes.keys():
        if nodes[k]['type'] == 'VNF':
            vnfs[k] = nodes[k]
            vnfs[k]['ifaces'] = {}
        elif nodes[k]['type'] == 'other_network' or nodes[k]['type'] == 'external_network':
            other_nets[k] = nodes[k]
            other_nets[k]['external']=True
        elif nodes[k]['type'] == 'network':
            other_nets[k] = nodes[k]
            other_nets[k]['external']=False


#1.2: Check that VNF are present at database table vnfs. Insert uuid, description and external interfaces
    for name,vnf in vnfs.items():
        where = {"OR": {"tenant_id": tenant_id, 'public': "true"}}
        error_text = ""
        error_pos = "'topology':'nodes':'" + name + "'"
        if 'vnf_id' in vnf:
            error_text += " 'vnf_id' " +  vnf['vnf_id']
            where['uuid'] = vnf['vnf_id']
        if 'VNF model' in vnf:
            error_text += " 'VNF model' " +  vnf['VNF model']
            where['name'] = vnf['VNF model']
        if len(where) == 1:
            raise NfvoException("Descriptor need a 'vnf_id' or 'VNF model' field at " + error_pos, httperrors.Bad_Request)

        vnf_db = mydb.get_rows(SELECT=('uuid','name','description'),
                               FROM='vnfs',
                               WHERE=where)
        if len(vnf_db)==0:
            raise NfvoException("unknown" + error_text + " at " + error_pos, httperrors.Not_Found)
        elif len(vnf_db)>1:
            raise NfvoException("more than one" + error_text + " at " + error_pos + " Concrete with 'vnf_id'", httperrors.Conflict)
        vnf['uuid']=vnf_db[0]['uuid']
        vnf['description']=vnf_db[0]['description']
        #get external interfaces
        ext_ifaces = mydb.get_rows(SELECT=('external_name as name','i.uuid as iface_uuid', 'i.type as type'),
            FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces as i on vms.uuid=i.vm_id',
            WHERE={'vnfs.uuid':vnf['uuid'], 'external_name<>': None} )
        for ext_iface in ext_ifaces:
            vnf['ifaces'][ ext_iface['name'] ] = {'uuid':ext_iface['iface_uuid'], 'type':ext_iface['type']}

#1.4 get list of connections
    conections = topo['topology']['connections']
    conections_list = []
    conections_list_name = []
    for k in conections.keys():
        if type(conections[k]['nodes'])==dict: #dict with node:iface pairs
            ifaces_list = conections[k]['nodes'].items()
        elif type(conections[k]['nodes'])==list: #list with dictionary
            ifaces_list=[]
            conection_pair_list = map(lambda x: x.items(), conections[k]['nodes'] )
            for k2 in conection_pair_list:
                ifaces_list += k2

        con_type = conections[k].get("type", "link")
        if con_type != "link":
            if k in other_nets:
                raise NfvoException("Format error. Reapeted network name at 'topology':'connections':'{}'".format(str(k)), httperrors.Bad_Request)
            other_nets[k] = {'external': False}
            if conections[k].get("graph"):
                other_nets[k]["graph"] =   conections[k]["graph"]
            ifaces_list.append( (k, None) )


        if con_type == "external_network":
            other_nets[k]['external'] = True
            if conections[k].get("model"):
                other_nets[k]["model"] =   conections[k]["model"]
            else:
                other_nets[k]["model"] =   k
        if con_type == "dataplane_net" or con_type == "bridge_net":
            other_nets[k]["model"] = con_type

        conections_list_name.append(k)
        conections_list.append(set(ifaces_list)) #from list to set to operate as a set (this conversion removes elements that are repeated in a list)
        #print set(ifaces_list)
    #check valid VNF and iface names
        for iface in ifaces_list:
            if iface[0] not in vnfs and iface[0] not in other_nets :
                raise NfvoException("format error. Invalid VNF name at 'topology':'connections':'{}':'nodes':'{}'".format(
                                                                                        str(k), iface[0]), httperrors.Not_Found)
            if iface[0] in vnfs and iface[1] not in vnfs[ iface[0] ]['ifaces']:
                raise NfvoException("format error. Invalid interface name at 'topology':'connections':'{}':'nodes':'{}':'{}'".format(
                                                                                        str(k), iface[0], iface[1]), httperrors.Not_Found)

#1.5 unify connections from the pair list to a consolidated list
    index=0
    while index < len(conections_list):
        index2 = index+1
        while index2 < len(conections_list):
            if len(conections_list[index] & conections_list[index2])>0: #common interface, join nets
                conections_list[index] |= conections_list[index2]
                del conections_list[index2]
                del conections_list_name[index2]
            else:
                index2 += 1
        conections_list[index] = list(conections_list[index])  # from set to list again
        index += 1
    #for k in conections_list:
    #    print k



#1.6 Delete non external nets
#    for k in other_nets.keys():
#        if other_nets[k]['model']=='bridge' or other_nets[k]['model']=='dataplane_net' or other_nets[k]['model']=='bridge_net':
#            for con in conections_list:
#                delete_indexes=[]
#                for index in range(0,len(con)):
#                    if con[index][0] == k: delete_indexes.insert(0,index) #order from higher to lower
#                for index in delete_indexes:
#                    del con[index]
#            del other_nets[k]
#1.7: Check external_ports are present at database table datacenter_nets
    for k,net in other_nets.items():
        error_pos = "'topology':'nodes':'" + k + "'"
        if net['external']==False:
            if 'name' not in net:
                net['name']=k
            if 'model' not in net:
                raise NfvoException("needed a 'model' at " + error_pos, httperrors.Bad_Request)
            if net['model']=='bridge_net':
                net['type']='bridge';
            elif net['model']=='dataplane_net':
                net['type']='data';
            else:
                raise NfvoException("unknown 'model' '"+ net['model'] +"' at " + error_pos, httperrors.Not_Found)
        else: #external
#IF we do not want to check that external network exist at datacenter
            pass
#ELSE
#             error_text = ""
#             WHERE_={}
#             if 'net_id' in net:
#                 error_text += " 'net_id' " +  net['net_id']
#                 WHERE_['uuid'] = net['net_id']
#             if 'model' in net:
#                 error_text += " 'model' " +  net['model']
#                 WHERE_['name'] = net['model']
#             if len(WHERE_) == 0:
#                 return -httperrors.Bad_Request, "needed a 'net_id' or 'model' at " + error_pos
#             r,net_db = mydb.get_table(SELECT=('uuid','name','description','type','shared'),
#                 FROM='datacenter_nets', WHERE=WHERE_ )
#             if r<0:
#                 print "nfvo.new_scenario Error getting datacenter_nets",r,net_db
#             elif r==0:
#                 print "nfvo.new_scenario Error" +error_text+ " is not present at database"
#                 return -httperrors.Bad_Request, "unknown " +error_text+ " at " + error_pos
#             elif r>1:
#                 print "nfvo.new_scenario Error more than one external_network for " +error_text+ " is present at database"
#                 return -httperrors.Bad_Request, "more than one external_network for " +error_text+ "at "+ error_pos + " Concrete with 'net_id'"
#             other_nets[k].update(net_db[0])
#ENDIF
    net_list={}
    net_nb=0  #Number of nets
    for con in conections_list:
        #check if this is connected to a external net
        other_net_index=-1
        #print
        #print "con", con
        for index in range(0,len(con)):
            #check if this is connected to a external net
            for net_key in other_nets.keys():
                if con[index][0]==net_key:
                    if other_net_index>=0:
                        error_text = "There is some interface connected both to net '{}' and net '{}'".format(
                            con[other_net_index][0], net_key)
                        #print "nfvo.new_scenario " + error_text
                        raise NfvoException(error_text, httperrors.Bad_Request)
                    else:
                        other_net_index = index
                        net_target = net_key
                    break
        #print "other_net_index", other_net_index
        try:
            if other_net_index>=0:
                del con[other_net_index]
#IF we do not want to check that external network exist at datacenter
                if other_nets[net_target]['external'] :
                    if "name" not in other_nets[net_target]:
                        other_nets[net_target]['name'] =  other_nets[net_target]['model']
                    if other_nets[net_target]["type"] == "external_network":
                        if vnfs[ con[0][0] ]['ifaces'][ con[0][1] ]["type"] == "data":
                            other_nets[net_target]["type"] =  "data"
                        else:
                            other_nets[net_target]["type"] =  "bridge"
#ELSE
#                 if other_nets[net_target]['external'] :
#                     type_='data' if len(con)>1 else 'ptp'  #an external net is connected to a external port, so it is ptp if only one connection is done to this net
#                     if type_=='data' and other_nets[net_target]['type']=="ptp":
#                         error_text = "Error connecting %d nodes on a not multipoint net %s" % (len(con), net_target)
#                         print "nfvo.new_scenario " + error_text
#                         return -httperrors.Bad_Request, error_text
#ENDIF
                for iface in con:
                    vnfs[ iface[0] ]['ifaces'][ iface[1] ]['net_key'] = net_target
            else:
                #create a net
                net_type_bridge=False
                net_type_data=False
                net_target = "__-__net"+str(net_nb)
                net_list[net_target] = {'name': conections_list_name[net_nb],  #"net-"+str(net_nb),
                    'description':"net-{} in scenario {}".format(net_nb,topo['name']),
                    'external':False}
                for iface in con:
                    vnfs[ iface[0] ]['ifaces'][ iface[1] ]['net_key'] = net_target
                    iface_type = vnfs[ iface[0] ]['ifaces'][ iface[1] ]['type']
                    if iface_type=='mgmt' or iface_type=='bridge':
                        net_type_bridge = True
                    else:
                        net_type_data = True
                if net_type_bridge and net_type_data:
                    error_text = "Error connection interfaces of bridge type with data type. Firs node {}, iface {}".format(iface[0], iface[1])
                    #print "nfvo.new_scenario " + error_text
                    raise NfvoException(error_text, httperrors.Bad_Request)
                elif net_type_bridge:
                    type_='bridge'
                else:
                    type_='data' if len(con)>2 else 'ptp'
                net_list[net_target]['type'] = type_
                net_nb+=1
        except Exception:
            error_text = "Error connection node {} : {} does not match any VNF or interface".format(iface[0], iface[1])
            #print "nfvo.new_scenario " + error_text
            #raise e
            raise NfvoException(error_text, httperrors.Bad_Request)

#1.8: Connect to management net all not already connected interfaces of type 'mgmt'
    #1.8.1 obtain management net
    mgmt_net = mydb.get_rows(SELECT=('uuid','name','description','type','shared'),
        FROM='datacenter_nets', WHERE={'name':'mgmt'} )
    #1.8.2 check all interfaces from all vnfs
    if len(mgmt_net)>0:
        add_mgmt_net = False
        for vnf in vnfs.values():
            for iface in vnf['ifaces'].values():
                if iface['type']=='mgmt' and 'net_key' not in iface:
                    #iface not connected
                    iface['net_key'] = 'mgmt'
                    add_mgmt_net = True
        if add_mgmt_net and 'mgmt' not in net_list:
            net_list['mgmt']=mgmt_net[0]
            net_list['mgmt']['external']=True
            net_list['mgmt']['graph']={'visible':False}

    net_list.update(other_nets)
    #print
    #print 'net_list', net_list
    #print
    #print 'vnfs', vnfs
    #print

#2: insert scenario. filling tables scenarios,sce_vnfs,sce_interfaces,sce_nets
    c = mydb.new_scenario( { 'vnfs':vnfs, 'nets':net_list,
        'tenant_id':tenant_id, 'name':topo['name'],
         'description':topo.get('description',topo['name']),
         'public': topo.get('public', False)
         })

    return c


@deprecated("Use new_nsd_v3")
def new_scenario_v02(mydb, tenant_id, scenario_dict, version):
    """ This creates a new scenario for version 0.2 and 0.3"""
    scenario = scenario_dict["scenario"]
    if tenant_id != "any":
        check_tenant(mydb, tenant_id)
        if "tenant_id" in scenario:
            if scenario["tenant_id"] != tenant_id:
                # print "nfvo.new_scenario_v02() tenant '%s' not found" % tenant_id
                raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(
                                                    scenario["tenant_id"], tenant_id), httperrors.Unauthorized)
    else:
        tenant_id=None

    # 1: Check that VNF are present at database table vnfs and update content into scenario dict
    for name,vnf in scenario["vnfs"].items():
        where = {"OR": {"tenant_id": tenant_id, 'public': "true"}}
        error_text = ""
        error_pos = "'scenario':'vnfs':'" + name + "'"
        if 'vnf_id' in vnf:
            error_text += " 'vnf_id' " + vnf['vnf_id']
            where['uuid'] = vnf['vnf_id']
        if 'vnf_name' in vnf:
            error_text += " 'vnf_name' " + vnf['vnf_name']
            where['name'] = vnf['vnf_name']
        if len(where) == 1:
            raise NfvoException("Needed a 'vnf_id' or 'vnf_name' at " + error_pos, httperrors.Bad_Request)
        vnf_db = mydb.get_rows(SELECT=('uuid', 'name', 'description'),
                               FROM='vnfs',
                               WHERE=where)
        if len(vnf_db) == 0:
            raise NfvoException("Unknown" + error_text + " at " + error_pos, httperrors.Not_Found)
        elif len(vnf_db) > 1:
            raise NfvoException("More than one" + error_text + " at " + error_pos + " Concrete with 'vnf_id'", httperrors.Conflict)
        vnf['uuid'] = vnf_db[0]['uuid']
        vnf['description'] = vnf_db[0]['description']
        vnf['ifaces'] = {}
        # get external interfaces
        ext_ifaces = mydb.get_rows(SELECT=('external_name as name', 'i.uuid as iface_uuid', 'i.type as type'),
                                   FROM='vnfs join vms on vnfs.uuid=vms.vnf_id join interfaces as i on vms.uuid=i.vm_id',
                                   WHERE={'vnfs.uuid':vnf['uuid'], 'external_name<>': None} )
        for ext_iface in ext_ifaces:
            vnf['ifaces'][ ext_iface['name'] ] = {'uuid':ext_iface['iface_uuid'], 'type': ext_iface['type']}
        # TODO? get internal-connections from db.nets and their profiles, and update scenario[vnfs][internal-connections] accordingly

    # 2: Insert net_key and ip_address at every vnf interface
    for net_name, net in scenario["networks"].items():
        net_type_bridge = False
        net_type_data = False
        for iface_dict in net["interfaces"]:
            if version == "0.2":
                temp_dict = iface_dict
                ip_address = None
            elif version == "0.3":
                temp_dict = {iface_dict["vnf"] : iface_dict["vnf_interface"]}
                ip_address = iface_dict.get('ip_address', None)
            for vnf, iface in temp_dict.items():
                if vnf not in scenario["vnfs"]:
                    error_text = "Error at 'networks':'{}':'interfaces' VNF '{}' not match any VNF at 'vnfs'".format(
                        net_name, vnf)
                    # logger.debug("nfvo.new_scenario_v02 " + error_text)
                    raise NfvoException(error_text, httperrors.Not_Found)
                if iface not in scenario["vnfs"][vnf]['ifaces']:
                    error_text = "Error at 'networks':'{}':'interfaces':'{}' interface not match any VNF interface"\
                        .format(net_name, iface)
                    # logger.debug("nfvo.new_scenario_v02 " + error_text)
                    raise NfvoException(error_text, httperrors.Bad_Request)
                if "net_key" in scenario["vnfs"][vnf]['ifaces'][iface]:
                    error_text = "Error at 'networks':'{}':'interfaces':'{}' interface already connected at network"\
                                 "'{}'".format(net_name, iface,scenario["vnfs"][vnf]['ifaces'][iface]['net_key'])
                    # logger.debug("nfvo.new_scenario_v02 " + error_text)
                    raise NfvoException(error_text, httperrors.Bad_Request)
                scenario["vnfs"][vnf]['ifaces'][ iface ]['net_key'] = net_name
                scenario["vnfs"][vnf]['ifaces'][iface]['ip_address'] = ip_address
                iface_type = scenario["vnfs"][vnf]['ifaces'][iface]['type']
                if iface_type == 'mgmt' or iface_type == 'bridge':
                    net_type_bridge = True
                else:
                    net_type_data = True

        if net_type_bridge and net_type_data:
            error_text = "Error connection interfaces of 'bridge' type and 'data' type at 'networks':'{}':'interfaces'"\
                .format(net_name)
            # logger.debug("nfvo.new_scenario " + error_text)
            raise NfvoException(error_text, httperrors.Bad_Request)
        elif net_type_bridge:
            type_ = 'bridge'
        else:
            type_ = 'data' if len(net["interfaces"]) > 2 else 'ptp'

        if net.get("implementation"):     # for v0.3
            if type_ == "bridge" and net["implementation"] == "underlay":
                error_text = "Error connecting interfaces of data type to a network declared as 'underlay' at "\
                             "'network':'{}'".format(net_name)
                # logger.debug(error_text)
                raise NfvoException(error_text, httperrors.Bad_Request)
            elif type_ != "bridge" and net["implementation"] == "overlay":
                error_text = "Error connecting interfaces of data type to a network declared as 'overlay' at "\
                             "'network':'{}'".format(net_name)
                # logger.debug(error_text)
                raise NfvoException(error_text, httperrors.Bad_Request)
            net.pop("implementation")
        if "type" in net and version == "0.3":   # for v0.3
            if type_ == "data" and net["type"] == "e-line":
                error_text = "Error connecting more than 2 interfaces of data type to a network declared as type "\
                             "'e-line' at 'network':'{}'".format(net_name)
                # logger.debug(error_text)
                raise NfvoException(error_text, httperrors.Bad_Request)
            elif type_ == "ptp" and net["type"] == "e-lan":
                type_ = "data"

        net['type'] = type_
        net['name'] = net_name
        net['external'] = net.get('external', False)

    # 3: insert at database
    scenario["nets"] = scenario["networks"]
    scenario['tenant_id'] = tenant_id
    scenario_id = mydb.new_scenario(scenario)
    return scenario_id


def new_nsd_v3(mydb, tenant_id, nsd_descriptor):
    """
    Parses an OSM IM nsd_catalog and insert at DB
    :param mydb:
    :param tenant_id:
    :param nsd_descriptor:
    :return: The list of created NSD ids
    """
    try:
        mynsd = nsd_catalog.nsd()
        try:
            pybindJSONDecoder.load_ietf_json(nsd_descriptor, None, None, obj=mynsd, skip_unknown=True)
        except Exception as e:
            raise NfvoException("Error. Invalid NS descriptor format: " + str(e), httperrors.Bad_Request)
        db_scenarios = []
        db_sce_nets = []
        db_sce_vnfs = []
        db_sce_interfaces = []
        db_sce_vnffgs = []
        db_sce_rsps = []
        db_sce_rsp_hops = []
        db_sce_classifiers = []
        db_sce_classifier_matches = []
        db_ip_profiles = []
        db_ip_profiles_index = 0
        uuid_list = []
        nsd_uuid_list = []
        for nsd_yang in mynsd.nsd_catalog.nsd.values():
            nsd = nsd_yang.get()

            # table scenarios
            scenario_uuid = str(uuid4())
            uuid_list.append(scenario_uuid)
            nsd_uuid_list.append(scenario_uuid)
            db_scenario = {
                "uuid": scenario_uuid,
                "osm_id": get_str(nsd, "id", 255),
                "name": get_str(nsd, "name", 255),
                "description": get_str(nsd, "description", 255),
                "tenant_id": tenant_id,
                "vendor": get_str(nsd, "vendor", 255),
                "short_name": get_str(nsd, "short-name", 255),
                "descriptor": str(nsd_descriptor)[:60000],
            }
            db_scenarios.append(db_scenario)

            # table sce_vnfs (constituent-vnfd)
            vnf_index2scevnf_uuid = {}
            vnf_index2vnf_uuid = {}
            for vnf in nsd.get("constituent-vnfd").values():
                existing_vnf = mydb.get_rows(FROM="vnfs", WHERE={'osm_id': str(vnf["vnfd-id-ref"])[:255],
                                                                      'tenant_id': tenant_id})
                if not existing_vnf:
                    raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'constituent-vnfd':'vnfd-id-ref':"
                                        "'{}'. Reference to a non-existing VNFD in the catalog".format(
                                            str(nsd["id"]), str(vnf["vnfd-id-ref"])[:255]),
                                        httperrors.Bad_Request)
                sce_vnf_uuid = str(uuid4())
                uuid_list.append(sce_vnf_uuid)
                db_sce_vnf = {
                    "uuid": sce_vnf_uuid,
                    "scenario_id": scenario_uuid,
                    # "name": get_str(vnf, "member-vnf-index", 255),
                    "name": existing_vnf[0]["name"][:200] + "." + get_str(vnf, "member-vnf-index", 50),
                    "vnf_id": existing_vnf[0]["uuid"],
                    "member_vnf_index": str(vnf["member-vnf-index"]),
                    # TODO 'start-by-default': True
                }
                vnf_index2scevnf_uuid[str(vnf['member-vnf-index'])] = sce_vnf_uuid
                vnf_index2vnf_uuid[str(vnf['member-vnf-index'])] = existing_vnf[0]["uuid"]
                db_sce_vnfs.append(db_sce_vnf)

            # table ip_profiles (ip-profiles)
            ip_profile_name2db_table_index = {}
            for ip_profile in nsd.get("ip-profiles").values():
                db_ip_profile = {
                    "ip_version": str(ip_profile["ip-profile-params"].get("ip-version", "ipv4")),
                    "subnet_address": str(ip_profile["ip-profile-params"].get("subnet-address")),
                    "gateway_address": str(ip_profile["ip-profile-params"].get("gateway-address")),
                    "dhcp_enabled": str(ip_profile["ip-profile-params"]["dhcp-params"].get("enabled", True)),
                    "dhcp_start_address": str(ip_profile["ip-profile-params"]["dhcp-params"].get("start-address")),
                    "dhcp_count": str(ip_profile["ip-profile-params"]["dhcp-params"].get("count")),
                }
                dns_list = []
                for dns in ip_profile["ip-profile-params"]["dns-server"].values():
                    dns_list.append(str(dns.get("address")))
                db_ip_profile["dns_address"] = ";".join(dns_list)
                if ip_profile["ip-profile-params"].get('security-group'):
                    db_ip_profile["security_group"] = ip_profile["ip-profile-params"]['security-group']
                ip_profile_name2db_table_index[str(ip_profile["name"])] = db_ip_profiles_index
                db_ip_profiles_index += 1
                db_ip_profiles.append(db_ip_profile)

            # table sce_nets (internal-vld)
            for vld in nsd.get("vld").values():
                sce_net_uuid = str(uuid4())
                uuid_list.append(sce_net_uuid)
                db_sce_net = {
                    "uuid": sce_net_uuid,
                    "name": get_str(vld, "name", 255),
                    "scenario_id": scenario_uuid,
                    # "type": #TODO
                    "multipoint": not vld.get("type") == "ELINE",
                    "osm_id":  get_str(vld, "id", 255),
                    # "external": #TODO
                    "description": get_str(vld, "description", 255),
                }
                # guess type of network
                if vld.get("mgmt-network"):
                    db_sce_net["type"] = "bridge"
                    db_sce_net["external"] = True
                elif vld.get("provider-network").get("overlay-type") == "VLAN":
                    db_sce_net["type"] = "data"
                else:
                    # later on it will be fixed to bridge or data depending on the type of interfaces attached to it
                    db_sce_net["type"] = None
                db_sce_nets.append(db_sce_net)

                # ip-profile, link db_ip_profile with db_sce_net
                if vld.get("ip-profile-ref"):
                    ip_profile_name = vld.get("ip-profile-ref")
                    if ip_profile_name not in ip_profile_name2db_table_index:
                        raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'ip-profile-ref':'{}'."
                                            " Reference to a non-existing 'ip_profiles'".format(
                                                str(nsd["id"]), str(vld["id"]), str(vld["ip-profile-ref"])),
                                            httperrors.Bad_Request)
                    db_ip_profiles[ip_profile_name2db_table_index[ip_profile_name]]["sce_net_id"] = sce_net_uuid
                elif vld.get("vim-network-name"):
                    db_sce_net["vim_network_name"] = get_str(vld, "vim-network-name", 255)

                # table sce_interfaces (vld:vnfd-connection-point-ref)
                for iface in vld.get("vnfd-connection-point-ref").values():
                    # Check if there are VDUs in the descriptor
                    vnf_index = str(iface['member-vnf-index-ref'])
                    existing_vdus = mydb.get_rows(SELECT=('vms.uuid'), FROM="vms", WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index]})
                    if existing_vdus:
                        # check correct parameters
                        if vnf_index not in vnf_index2vnf_uuid:
                            raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'vnfd-connection-point"
                                              "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at "
                                              "'nsd':'constituent-vnfd'".format(
                                                  str(nsd["id"]), str(vld["id"]), str(iface["member-vnf-index-ref"])),
                                              httperrors.Bad_Request)

                        existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid', 'i.type as iface_type'),
                                                      FROM="interfaces as i join vms on i.vm_id=vms.uuid",
                                                      WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index],
                                                             'external_name': get_str(iface, "vnfd-connection-point-ref",
                                                                                      255)})
                        if not existing_ifaces:
                            raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'vnfd-connection-point"
                                              "-ref':'vnfd-connection-point-ref':'{}'. Reference to a non-existing "
                                              "connection-point name at VNFD '{}'".format(
                                                  str(nsd["id"]), str(vld["id"]), str(iface["vnfd-connection-point-ref"]),
                                                  str(iface.get("vnfd-id-ref"))[:255]),
                                              httperrors.Bad_Request)
                        interface_uuid = existing_ifaces[0]["uuid"]
                        if existing_ifaces[0]["iface_type"] == "data":
                            db_sce_net["type"] = "data"
                        sce_interface_uuid = str(uuid4())
                        uuid_list.append(sce_net_uuid)
                        iface_ip_address = None
                        if iface.get("ip-address"):
                            iface_ip_address = str(iface.get("ip-address"))
                        db_sce_interface = {
                            "uuid": sce_interface_uuid,
                            "sce_vnf_id": vnf_index2scevnf_uuid[vnf_index],
                            "sce_net_id": sce_net_uuid,
                            "interface_id": interface_uuid,
                            "ip_address": iface_ip_address,
                        }
                        db_sce_interfaces.append(db_sce_interface)
                        if not db_sce_net["type"]:
                            db_sce_net["type"] = "bridge"

            # table sce_vnffgs (vnffgd)
            for vnffg in nsd.get("vnffgd").values():
                sce_vnffg_uuid = str(uuid4())
                uuid_list.append(sce_vnffg_uuid)
                db_sce_vnffg = {
                    "uuid": sce_vnffg_uuid,
                    "name": get_str(vnffg, "name", 255),
                    "scenario_id": scenario_uuid,
                    "vendor": get_str(vnffg, "vendor", 255),
                    "description": get_str(vld, "description", 255),
                }
                db_sce_vnffgs.append(db_sce_vnffg)

                # deal with rsps
                for rsp in vnffg.get("rsp").values():
                    sce_rsp_uuid = str(uuid4())
                    uuid_list.append(sce_rsp_uuid)
                    db_sce_rsp = {
                        "uuid": sce_rsp_uuid,
                        "name": get_str(rsp, "name", 255),
                        "sce_vnffg_id": sce_vnffg_uuid,
                        "id": get_str(rsp, "id", 255), # only useful to link with classifiers; will be removed later in the code
                    }
                    db_sce_rsps.append(db_sce_rsp)
                    for iface in rsp.get("vnfd-connection-point-ref").values():
                        vnf_index = str(iface['member-vnf-index-ref'])
                        if_order = int(iface['order'])
                        # check correct parameters
                        if vnf_index not in vnf_index2vnf_uuid:
                            raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point"
                                                "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at "
                                                "'nsd':'constituent-vnfd'".format(
                                                    str(nsd["id"]), str(rsp["id"]), str(iface["member-vnf-index-ref"])),
                                                httperrors.Bad_Request)

                        ingress_existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',),
                                                                FROM="interfaces as i join vms on i.vm_id=vms.uuid",
                                                                WHERE={
                                                                    'vnf_id': vnf_index2vnf_uuid[vnf_index],
                                                                    'external_name': get_str(iface, "vnfd-ingress-connection-point-ref",
                                                                                             255)})
                        if not ingress_existing_ifaces:
                            raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point"
                                                "-ref':'vnfd-ingress-connection-point-ref':'{}'. Reference to a non-existing "
                                                "connection-point name at VNFD '{}'".format(
                                str(nsd["id"]), str(rsp["id"]), str(iface["vnfd-ingress-connection-point-ref"]),
                                str(iface.get("vnfd-id-ref"))[:255]), httperrors.Bad_Request)

                        egress_existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',),
                                                               FROM="interfaces as i join vms on i.vm_id=vms.uuid",
                                                               WHERE={
                                                                   'vnf_id': vnf_index2vnf_uuid[vnf_index],
                                                                   'external_name': get_str(iface, "vnfd-egress-connection-point-ref",
                                                                                            255)})
                        if not egress_existing_ifaces:
                            raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point"
                                                "-ref':'vnfd-egress-connection-point-ref':'{}'. Reference to a non-existing "
                                                "connection-point name at VNFD '{}'".format(
                                str(nsd["id"]), str(rsp["id"]), str(iface["vnfd-egress-connection-point-ref"]),
                                str(iface.get("vnfd-id-ref"))[:255]), HTTP_Bad_Request)

                        ingress_interface_uuid = ingress_existing_ifaces[0]["uuid"]
                        egress_interface_uuid = egress_existing_ifaces[0]["uuid"]
                        sce_rsp_hop_uuid = str(uuid4())
                        uuid_list.append(sce_rsp_hop_uuid)
                        db_sce_rsp_hop = {
                            "uuid": sce_rsp_hop_uuid,
                            "if_order": if_order,
                            "ingress_interface_id": ingress_interface_uuid,
                            "egress_interface_id": egress_interface_uuid,
                            "sce_vnf_id": vnf_index2scevnf_uuid[vnf_index],
                            "sce_rsp_id": sce_rsp_uuid,
                        }
                        db_sce_rsp_hops.append(db_sce_rsp_hop)

                # deal with classifiers
                for classifier in vnffg.get("classifier").values():
                    sce_classifier_uuid = str(uuid4())
                    uuid_list.append(sce_classifier_uuid)

                    # source VNF
                    vnf_index = str(classifier['member-vnf-index-ref'])
                    if vnf_index not in vnf_index2vnf_uuid:
                        raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'classifier[{}]':'vnfd-connection-point"
                                            "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at "
                                            "'nsd':'constituent-vnfd'".format(
                                                str(nsd["id"]), str(classifier["id"]), str(classifier["member-vnf-index-ref"])),
                                            httperrors.Bad_Request)
                    existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',),
                                                    FROM="interfaces as i join vms on i.vm_id=vms.uuid",
                                                    WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index],
                                                           'external_name': get_str(classifier, "vnfd-connection-point-ref",
                                                                                    255)})
                    if not existing_ifaces:
                        raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'rsp[{}]':'vnfd-connection-point"
                                            "-ref':'vnfd-connection-point-ref':'{}'. Reference to a non-existing "
                                            "connection-point name at VNFD '{}'".format(
                                                str(nsd["id"]), str(rsp["id"]), str(iface["vnfd-connection-point-ref"]),
                                                str(iface.get("vnfd-id-ref"))[:255]),
                                            httperrors.Bad_Request)
                    interface_uuid = existing_ifaces[0]["uuid"]

                    db_sce_classifier = {
                        "uuid": sce_classifier_uuid,
                        "name": get_str(classifier, "name", 255),
                        "sce_vnffg_id": sce_vnffg_uuid,
                        "sce_vnf_id": vnf_index2scevnf_uuid[vnf_index],
                        "interface_id": interface_uuid,
                    }
                    rsp_id = get_str(classifier, "rsp-id-ref", 255)
                    rsp = next((item for item in db_sce_rsps if item["id"] == rsp_id), None)
                    db_sce_classifier["sce_rsp_id"] = rsp["uuid"]
                    db_sce_classifiers.append(db_sce_classifier)

                    for match in classifier.get("match-attributes").values():
                        sce_classifier_match_uuid = str(uuid4())
                        uuid_list.append(sce_classifier_match_uuid)
                        db_sce_classifier_match = {
                            "uuid": sce_classifier_match_uuid,
                            "ip_proto": get_str(match, "ip-proto", 2),
                            "source_ip": get_str(match, "source-ip-address", 16),
                            "destination_ip": get_str(match, "destination-ip-address", 16),
                            "source_port": get_str(match, "source-port", 5),
                            "destination_port": get_str(match, "destination-port", 5),
                            "sce_classifier_id": sce_classifier_uuid,
                        }
                        db_sce_classifier_matches.append(db_sce_classifier_match)
                    # TODO: vnf/cp keys

        # remove unneeded id's in sce_rsps
        for rsp in db_sce_rsps:
            rsp.pop('id')

        db_tables = [
            {"scenarios": db_scenarios},
            {"sce_nets": db_sce_nets},
            {"ip_profiles": db_ip_profiles},
            {"sce_vnfs": db_sce_vnfs},
            {"sce_interfaces": db_sce_interfaces},
            {"sce_vnffgs": db_sce_vnffgs},
            {"sce_rsps": db_sce_rsps},
            {"sce_rsp_hops": db_sce_rsp_hops},
            {"sce_classifiers": db_sce_classifiers},
            {"sce_classifier_matches": db_sce_classifier_matches},
        ]

        logger.debug("new_nsd_v3 done: %s",
                    yaml.safe_dump(db_tables, indent=4, default_flow_style=False) )
        mydb.new_rows(db_tables, uuid_list)
        return nsd_uuid_list
    except NfvoException:
        raise
    except Exception as e:
        logger.error("Exception {}".format(e))
        raise  # NfvoException("Exception {}".format(e), httperrors.Bad_Request)


def edit_scenario(mydb, tenant_id, scenario_id, data):
    data["uuid"] = scenario_id
    data["tenant_id"] = tenant_id
    c = mydb.edit_scenario( data )
    return c


@deprecated("Use create_instance")
def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instance_scenario_description, datacenter=None,vim_tenant=None, startvms=True):
    #print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id"
    datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter, vim_tenant=vim_tenant)
    vims = {datacenter_id: myvim}
    myvim_tenant = myvim['tenant_id']
    datacenter_name = myvim['name']

    rollbackList=[]
    try:
        #print "Checking that the scenario_id exists and getting the scenario dictionary"
        scenarioDict = mydb.get_scenario(scenario_id, tenant_id, datacenter_id=datacenter_id)
        scenarioDict['datacenter2tenant'] = { datacenter_id: myvim['config']['datacenter_tenant_id'] }
        scenarioDict['datacenter_id'] = datacenter_id
        #print '================scenarioDict======================='
        #print json.dumps(scenarioDict, indent=4)
        #print 'BEGIN launching instance scenario "%s" based on "%s"' % (instance_scenario_name,scenarioDict['name'])

        logger.debug("start_scenario Scenario %s: consisting of %d VNF(s)", scenarioDict['name'],len(scenarioDict['vnfs']))
        #print yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)

        auxNetDict = {}   #Auxiliar dictionary. First key:'scenario' or sce_vnf uuid. Second Key: uuid of the net/sce_net. Value: vim_net_id
        auxNetDict['scenario'] = {}

        logger.debug("start_scenario 1. Creating new nets (sce_nets) in the VIM")
        for sce_net in scenarioDict['nets']:
            #print "Net name: %s. Description: %s" % (sce_net["name"], sce_net["description"])

            myNetName = "{}.{}".format(instance_scenario_name, sce_net['name'])
            myNetName = myNetName[0:255] #limit length
            myNetType = sce_net['type']
            myNetDict = {}
            myNetDict["name"] = myNetName
            myNetDict["type"] = myNetType
            myNetDict["tenant_id"] = myvim_tenant
            myNetIPProfile = sce_net.get('ip_profile', None)
            myProviderNetwork = sce_net.get('provider_network', None)
            #TODO:
            #We should use the dictionary as input parameter for new_network
            #print myNetDict
            if not sce_net["external"]:
                network_id, _ = myvim.new_network(myNetName, myNetType, myNetIPProfile, provider_network_profile=myProviderNetwork)
                #print "New VIM network created for scenario %s. Network id:  %s" % (scenarioDict['name'],network_id)
                sce_net['vim_id'] = network_id
                auxNetDict['scenario'][sce_net['uuid']] = network_id
                rollbackList.append({'what':'network','where':'vim','vim_id':datacenter_id,'uuid':network_id})
                sce_net["created"] = True
            else:
                if sce_net['vim_id'] == None:
                    error_text = "Error, datacenter '{}' does not have external network '{}'.".format(
                        datacenter_name, sce_net['name'])
                    _, message = rollback(mydb, vims, rollbackList)
                    logger.error("nfvo.start_scenario: %s", error_text)
                    raise NfvoException(error_text, httperrors.Bad_Request)
                logger.debug("Using existent VIM network for scenario %s. Network id %s", scenarioDict['name'],sce_net['vim_id'])
                auxNetDict['scenario'][sce_net['uuid']] = sce_net['vim_id']

        logger.debug("start_scenario 2. Creating new nets (vnf internal nets) in the VIM")
        #For each vnf net, we create it and we add it to instanceNetlist.

        for sce_vnf in scenarioDict['vnfs']:
            for net in sce_vnf['nets']:
                #print "Net name: %s. Description: %s" % (net["name"], net["description"])

                myNetName = "{}.{}".format(instance_scenario_name,net['name'])
                myNetName = myNetName[0:255] #limit length
                myNetType = net['type']
                myNetDict = {}
                myNetDict["name"] = myNetName
                myNetDict["type"] = myNetType
                myNetDict["tenant_id"] = myvim_tenant
                myNetIPProfile = net.get('ip_profile', None)
                myProviderNetwork = sce_net.get('provider_network', None)
                #print myNetDict
                #TODO:
                #We should use the dictionary as input parameter for new_network
                network_id, _  = myvim.new_network(myNetName, myNetType, myNetIPProfile, provider_network_profile=myProviderNetwork)
                #print "VIM network id for scenario %s: %s" % (scenarioDict['name'],network_id)
                net['vim_id'] = network_id
                if sce_vnf['uuid'] not in auxNetDict:
                    auxNetDict[sce_vnf['uuid']] = {}
                auxNetDict[sce_vnf['uuid']][net['uuid']] = network_id
                rollbackList.append({'what':'network','where':'vim','vim_id':datacenter_id,'uuid':network_id})
                net["created"] = True

        #print "auxNetDict:"
        #print yaml.safe_dump(auxNetDict, indent=4, default_flow_style=False)

        logger.debug("start_scenario 3. Creating new vm instances in the VIM")
        #myvim.new_vminstance(self,vimURI,tenant_id,name,description,image_id,flavor_id,net_dict)
        i = 0
        for sce_vnf in scenarioDict['vnfs']:
            vnf_availability_zones = []
            for vm in sce_vnf['vms']:
                vm_av = vm.get('availability_zone')
                if vm_av and vm_av not in vnf_availability_zones:
                    vnf_availability_zones.append(vm_av)

            # check if there is enough availability zones available at vim level.
            if myvims[datacenter_id].availability_zone and vnf_availability_zones:
                if len(vnf_availability_zones) > len(myvims[datacenter_id].availability_zone):
                    raise NfvoException('No enough availability zones at VIM for this deployment', httperrors.Bad_Request)

            for vm in sce_vnf['vms']:
                i += 1
                myVMDict = {}
                #myVMDict['name'] = "%s-%s-%s" % (scenarioDict['name'],sce_vnf['name'], vm['name'])
                myVMDict['name'] = "{}.{}.{}".format(instance_scenario_name,sce_vnf['name'],chr(96+i))
                #myVMDict['description'] = vm['description']
                myVMDict['description'] = myVMDict['name'][0:99]
                if not startvms:
                    myVMDict['start'] = "no"
                myVMDict['name'] = myVMDict['name'][0:255] #limit name length
                #print "VM name: %s. Description: %s" % (myVMDict['name'], myVMDict['name'])

                #create image at vim in case it not exist
                image_dict = mydb.get_table_by_uuid_name("images", vm['image_id'])
                image_id = create_or_use_image(mydb, vims, image_dict, [], True)
                vm['vim_image_id'] = image_id

                #create flavor at vim in case it not exist
                flavor_dict = mydb.get_table_by_uuid_name("flavors", vm['flavor_id'])
                if flavor_dict['extended']!=None:
                    flavor_dict['extended']= yaml.load(flavor_dict['extended'], Loader=yaml.Loader)
                flavor_id = create_or_use_flavor(mydb, vims, flavor_dict, [], True)
                vm['vim_flavor_id'] = flavor_id


                myVMDict['imageRef'] = vm['vim_image_id']
                myVMDict['flavorRef'] = vm['vim_flavor_id']
                myVMDict['networks'] = []
                for iface in vm['interfaces']:
                    netDict = {}
                    if iface['type']=="data":
                        netDict['type'] = iface['model']
                    elif "model" in iface and iface["model"]!=None:
                        netDict['model']=iface['model']
                    #TODO in future, remove this because mac_address will not be set, and the type of PV,VF is obtained from iterface table model
                    #discover type of interface looking at flavor
                    for numa in flavor_dict.get('extended',{}).get('numas',[]):
                        for flavor_iface in numa.get('interfaces',[]):
                            if flavor_iface.get('name') == iface['internal_name']:
                                if flavor_iface['dedicated'] == 'yes':
                                    netDict['type']="PF"    #passthrough
                                elif flavor_iface['dedicated'] == 'no':
                                    netDict['type']="VF"    #siov
                                elif flavor_iface['dedicated'] == 'yes:sriov':
                                    netDict['type']="VFnotShared"   #sriov but only one sriov on the PF
                                netDict["mac_address"] = flavor_iface.get("mac_address")
                                break;
                    netDict["use"]=iface['type']
                    if netDict["use"]=="data" and not netDict.get("type"):
                        #print "netDict", netDict
                        #print "iface", iface
                        e_text = "Cannot determine the interface type PF or VF of VNF '{}' VM '{}' iface '{}'".format(
                            sce_vnf['name'], vm['name'], iface['internal_name'])
                        if flavor_dict.get('extended')==None:
                            raise NfvoException(e_text  + "After database migration some information is not available. \
                                    Try to delete and create the scenarios and VNFs again", httperrors.Conflict)
                        else:
                            raise NfvoException(e_text, httperrors.Internal_Server_Error)
                    if netDict["use"]=="mgmt" or netDict["use"]=="bridge":
                        netDict["type"]="virtual"
                    if "vpci" in iface and iface["vpci"] is not None:
                        netDict['vpci'] = iface['vpci']
                    if "mac" in iface and iface["mac"] is not None:
                        netDict['mac_address'] = iface['mac']
                    if "port-security" in iface and iface["port-security"] is not None:
                        netDict['port_security'] = iface['port-security']
                    if "floating-ip" in iface and iface["floating-ip"] is not None:
                        netDict['floating_ip'] = iface['floating-ip']
                    netDict['name'] = iface['internal_name']
                    if iface['net_id'] is None:
                        for vnf_iface in sce_vnf["interfaces"]:
                            #print iface
                            #print vnf_iface
                            if vnf_iface['interface_id']==iface['uuid']:
                                netDict['net_id'] = auxNetDict['scenario'][ vnf_iface['sce_net_id'] ]
                                break
                    else:
                        netDict['net_id'] = auxNetDict[ sce_vnf['uuid'] ][ iface['net_id'] ]
                    #skip bridge ifaces not connected to any net
                    #if 'net_id' not in netDict or netDict['net_id']==None:
                    #    continue
                    myVMDict['networks'].append(netDict)
                #print ">>>>>>>>>>>>>>>>>>>>>>>>>>>"
                #print myVMDict['name']
                #print "networks", yaml.safe_dump(myVMDict['networks'], indent=4, default_flow_style=False)
                #print "interfaces", yaml.safe_dump(vm['interfaces'], indent=4, default_flow_style=False)
                #print ">>>>>>>>>>>>>>>>>>>>>>>>>>>"

                if 'availability_zone' in myVMDict:
                    av_index = vnf_availability_zones.index(myVMDict['availability_zone'])
                else:
                    av_index = None

                vm_id, _ = myvim.new_vminstance(myVMDict['name'], myVMDict['description'], myVMDict.get('start', None),
                                             myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'],
                                             availability_zone_index=av_index,
                                             availability_zone_list=vnf_availability_zones)
                #print "VIM vm instance id (server id) for scenario %s: %s" % (scenarioDict['name'],vm_id)
                vm['vim_id'] = vm_id
                rollbackList.append({'what':'vm','where':'vim','vim_id':datacenter_id,'uuid':vm_id})
                #put interface uuid back to scenario[vnfs][vms[[interfaces]
                for net in myVMDict['networks']:
                    if "vim_id" in net:
                        for iface in vm['interfaces']:
                            if net["name"]==iface["internal_name"]:
                                iface["vim_id"]=net["vim_id"]
                                break

        logger.debug("start scenario Deployment done")
        #print yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)
        #r,c = mydb.new_instance_scenario_as_a_whole(nfvo_tenant,scenarioDict['name'],scenarioDict)
        instance_id = mydb.new_instance_scenario_as_a_whole(tenant_id,instance_scenario_name, instance_scenario_description, scenarioDict)
        return mydb.get_instance_scenario(instance_id)

    except (db_base_Exception, vimconn.VimConnException) as e:
        _, message = rollback(mydb, vims, rollbackList)
        if isinstance(e, db_base_Exception):
            error_text = "Exception at database"
        else:
            error_text = "Exception at VIM"
        error_text += " {} {}. {}".format(type(e).__name__, str(e), message)
        #logger.error("start_scenario %s", error_text)
        raise NfvoException(error_text, e.http_code)

def unify_cloud_config(cloud_config_preserve, cloud_config):
    """ join the cloud config information into cloud_config_preserve.
    In case of conflict cloud_config_preserve preserves
    None is allowed
    """
    if not cloud_config_preserve and not cloud_config:
        return None

    new_cloud_config = {"key-pairs":[], "users":[]}
    # key-pairs
    if cloud_config_preserve:
        for key in cloud_config_preserve.get("key-pairs", () ):
            if key not in new_cloud_config["key-pairs"]:
                new_cloud_config["key-pairs"].append(key)
    if cloud_config:
        for key in cloud_config.get("key-pairs", () ):
            if key not in new_cloud_config["key-pairs"]:
                new_cloud_config["key-pairs"].append(key)
    if not new_cloud_config["key-pairs"]:
        del new_cloud_config["key-pairs"]

    # users
    if cloud_config:
        new_cloud_config["users"] += cloud_config.get("users", () )
    if cloud_config_preserve:
        new_cloud_config["users"] += cloud_config_preserve.get("users", () )
    index_to_delete = []
    users = new_cloud_config.get("users", [])
    for index0 in range(0,len(users)):
        if index0 in index_to_delete:
            continue
        for index1 in range(index0+1,len(users)):
            if index1 in index_to_delete:
                continue
            if users[index0]["name"] == users[index1]["name"]:
                index_to_delete.append(index1)
                for key in users[index1].get("key-pairs",()):
                    if "key-pairs" not in users[index0]:
                        users[index0]["key-pairs"] = [key]
                    elif key not in users[index0]["key-pairs"]:
                        users[index0]["key-pairs"].append(key)
    index_to_delete.sort(reverse=True)
    for index in index_to_delete:
        del users[index]
    if not new_cloud_config["users"]:
        del new_cloud_config["users"]

    #boot-data-drive
    if cloud_config and cloud_config.get("boot-data-drive") != None:
        new_cloud_config["boot-data-drive"] = cloud_config["boot-data-drive"]
    if cloud_config_preserve and cloud_config_preserve.get("boot-data-drive") != None:
        new_cloud_config["boot-data-drive"] = cloud_config_preserve["boot-data-drive"]

    # user-data
    new_cloud_config["user-data"] = []
    if cloud_config and cloud_config.get("user-data"):
        if isinstance(cloud_config["user-data"], list):
            new_cloud_config["user-data"] += cloud_config["user-data"]
        else:
            new_cloud_config["user-data"].append(cloud_config["user-data"])
    if cloud_config_preserve and cloud_config_preserve.get("user-data"):
        if isinstance(cloud_config_preserve["user-data"], list):
            new_cloud_config["user-data"] += cloud_config_preserve["user-data"]
        else:
            new_cloud_config["user-data"].append(cloud_config_preserve["user-data"])
    if not new_cloud_config["user-data"]:
        del new_cloud_config["user-data"]

    # config files
    new_cloud_config["config-files"] = []
    if cloud_config and cloud_config.get("config-files") != None:
        new_cloud_config["config-files"] += cloud_config["config-files"]
    if cloud_config_preserve:
        for file in cloud_config_preserve.get("config-files", ()):
            for index in range(0, len(new_cloud_config["config-files"])):
                if new_cloud_config["config-files"][index]["dest"] == file["dest"]:
                    new_cloud_config["config-files"][index] = file
                    break
            else:
                new_cloud_config["config-files"].append(file)
    if not new_cloud_config["config-files"]:
        del new_cloud_config["config-files"]
    return new_cloud_config


def get_vim_thread(mydb, tenant_id, datacenter_id_name=None, datacenter_tenant_id=None):
    global plugins
    datacenter_id = None
    datacenter_name = None
    thread = None
    try:
        if datacenter_tenant_id:
            thread_id = datacenter_tenant_id
            thread = vim_threads["running"].get(datacenter_tenant_id)
        if not thread:
            where_={"td.nfvo_tenant_id": tenant_id}
            if datacenter_id_name:
                if utils.check_valid_uuid