bug 544 Adding license headers
[osm/LCM.git] / osm_lcm / ns.py
index 8f8cdac..6589e22 100644 (file)
@@ -1,6 +1,21 @@
-#!/usr/bin/python3
 # -*- coding: utf-8 -*-
 
+##
+# Copyright 2018 Telefonica S.A.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+##
+
 import asyncio
 import yaml
 import logging
@@ -9,7 +24,7 @@ import functools
 import traceback
 
 import ROclient
-from lcm_utils import LcmException, LcmBase
+from lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
 
 from osm_common.dbbase import DbException
 from osm_common.fsbase import FsException
@@ -52,6 +67,8 @@ def populate_dict(target_dict, key_list, value):
 
 
 class NsLcm(LcmBase):
+    timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
+    total_deploy_timeout = 2 * 3600   # global timeout for deployment
 
     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
         """
@@ -81,58 +98,6 @@ class NsLcm(LcmBase):
             artifacts=None,
         )
 
-    def _look_for_pdu(self, vdur, vnfr, vim_account):  # TODO feature 1417: project_id
-        """
-        Look for a free PDU in the catalog matching vdur type and interfaces. Fills vdur with ip_address information
-        :param vdur: vnfr:vdur descriptor. It is modified with pdu interface info if pdu is found
-        :param member_vnf_index: used just for logging. Target vnfd of nsd
-        :param vim_account:
-        :return: vder_update: dictionary to update vnfr:vdur with pdu info. In addition it modified choosen pdu to set
-        at status IN_USE
-        """
-        pdu_type = vdur.get("pdu-type")
-        assert pdu_type
-        pdu_filter = {
-            "vim.vim_accounts": vim_account,
-            "type": pdu_type,
-            "_admin.operationalState": "ENABLED",
-            "_admin.usageSate": "NOT_IN_USE",
-            # TODO feature 1417: "shared": True,
-            # TODO feature 1417:  "_admin.projects_read.cont": ["ANY", project_id],
-        }
-        available_pdus = self.db.get_list("pdus", pdu_filter)
-        for pdu in available_pdus:
-            # step 1 check if this pdu contains needed interfaces:
-            match_interfaces = True
-            for vdur_interface in vdur["interfaces"]:
-                for pdu_interface in pdu["interfaces"]:
-                    if pdu_interface["name"] == vdur_interface["name"]:
-                        # TODO feature 1417: match per mgmt type
-                        break
-                else:  # no interface found
-                    match_interfaces = False
-                    break
-            if not match_interfaces:
-                continue
-
-            # step 2. Update pdu
-            self.update_db_2("pdus", pdu["_id"], {"_admin.usageSate": "IN_USE",
-                                                  "_admin.usage.vnfr_id": vnfr["_id"],
-                                                  "_admin.usage.nsr_id": vnfr["nsr-id-ref"],
-                                                  "_admin.usage.vdur": vdur["vdu-id-ref"],
-                                                  })
-            # step 3. Fill vnfr info by filling vdur
-            vdur["pdu-id"] = pdu["_id"]
-            for vdur_interface in vdur["interfaces"]:
-                for pdu_interface in pdu["interfaces"]:
-                    if pdu_interface["name"] == vdur_interface["name"]:
-                        vdur_interface.update(pdu_interface)
-                        break
-
-            return
-        raise LcmException("No PDU of type={} found for member_vng_index={} at vim_account={} matching interface "
-                           "names".format(vdur["vdu-id-ref"], vnfr["member-vnf-index-ref"], pdu_type))
-
     def vnfd2RO(self, vnfd, new_id=None):
         """
         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
@@ -147,7 +112,7 @@ class NsLcm(LcmBase):
             vnfd_RO.pop("_admin", None)
             if new_id:
                 vnfd_RO["id"] = new_id
-            for vdu in vnfd_RO["vdu"]:
+            for vdu in vnfd_RO.get("vdu", ()):
                 if "cloud-init-file" in vdu:
                     base_folder = vnfd["_admin"]["storage"]
                     clout_init_file = "{}/{}/cloud_init/{}".format(
@@ -208,27 +173,30 @@ class NsLcm(LcmBase):
             n2vc_event = n2vc_info["n2vc_event"]
             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
                                                                                     application_name)
-            vca_deployed = deployed.get(application_name)
-            if not vca_deployed:
+            for vca_index, vca_deployed in enumerate(deployed):
+                if not vca_deployed:
+                    continue
+                if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
+                    break
+            else:
                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA")
                 return
-
             if task:
                 if task.cancelled():
                     self.logger.debug(logging_text + " task Cancelled")
                     vca_deployed['operational-status'] = "error"
-                    db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(application_name)] = "error"
+                    db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
                     vca_deployed['detailed-status'] = "Task Cancelled"
-                    db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(application_name)] = "Task Cancelled"
+                    db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 
                 elif task.done():
                     exc = task.exception()
                     if exc:
                         self.logger.error(logging_text + " task Exception={}".format(exc))
                         vca_deployed['operational-status'] = "error"
-                        db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(application_name)] = "error"
+                        db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
                         vca_deployed['detailed-status'] = str(exc)
-                        db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(application_name)] = str(exc)
+                        db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
                     else:
                         self.logger.debug(logging_text + " task Done")
                         # task is Done, but callback is still ongoing. So ignore
@@ -238,9 +206,9 @@ class NsLcm(LcmBase):
                 if vca_deployed['operational-status'] == status:
                     return  # same status, ignore
                 vca_deployed['operational-status'] = status
-                db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(application_name)] = status
+                db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
                 vca_deployed['detailed-status'] = str(message)
-                db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(application_name)] = str(message)
+                db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
             else:
                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
                 return
@@ -308,7 +276,7 @@ class NsLcm(LcmBase):
                         elif vnfd["mgmt-interface"].get("cp"):
                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 
-                for vdu in vnfd.get("vdu"):
+                for vdu in vnfd.get("vdu", ()):
                     if vdu.get("vdu-configuration"):
                         vdu_needed_access.append(vdu["id"])
                     elif mgmt_cp:
@@ -502,6 +470,28 @@ class NsLcm(LcmBase):
         db_vnfr["vdur"] = vdurs
         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 
+    def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
+        """
+        Updates database nsr with the RO info for the created vld
+        :param ns_update_nsr: dictionary to be filled with the updated info
+        :param db_nsr: content of db_nsr. This is also modified
+        :param nsr_desc_RO: nsr descriptor from RO
+        :return: Nothing, LcmException is raised on errors
+        """
+
+        for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
+            for net_RO in get_iterable(nsr_desc_RO, "nets"):
+                if vld["id"] != net_RO.get("ns_net_osm_id"):
+                    continue
+                vld["vim-id"] = net_RO.get("vim_net_id")
+                vld["name"] = net_RO.get("vim_name")
+                vld["status"] = net_RO.get("status")
+                vld["status-detailed"] = net_RO.get("error_msg")
+                ns_update_nsr["vld.{}".format(vld_index)] = vld
+                break
+            else:
+                raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
+
     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
         """
         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
@@ -514,7 +504,10 @@ class NsLcm(LcmBase):
                 if vnf_RO["member_vnf_index"] != vnf_index:
                     continue
                 vnfr_update = {}
-                db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO.get("ip_address")
+                if vnf_RO.get("ip_address"):
+                    db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"]
+                elif not db_vnfr.get("ip-address"):
+                    raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 
                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
                     vdur_RO_count_index = 0
@@ -546,6 +539,21 @@ class NsLcm(LcmBase):
                     else:
                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} at "
                                            "RO info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
+
+                for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
+                    for net_RO in get_iterable(nsr_desc_RO, "nets"):
+                        if vld["id"] != net_RO.get("vnf_net_osm_id"):
+                            continue
+                        vld["vim-id"] = net_RO.get("vim_net_id")
+                        vld["name"] = net_RO.get("vim_name")
+                        vld["status"] = net_RO.get("status")
+                        vld["status-detailed"] = net_RO.get("error_msg")
+                        vnfr_update["vld.{}".format(vld_index)] = vld
+                        break
+                    else:
+                        raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} at RO info".format(
+                            vnf_index, vld["id"]))
+
                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
                 break
 
@@ -556,6 +564,7 @@ class NsLcm(LcmBase):
         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
         self.logger.debug(logging_text + "Enter")
         # get all needed from database
+        start_deploy = time()
         db_nsr = None
         db_nslcmop = None
         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
@@ -571,7 +580,7 @@ class NsLcm(LcmBase):
             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
             step = "Getting nsr={} from db".format(nsr_id)
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
-            ns_params = db_nsr.get("instantiate_params")
+            ns_params = db_nslcmop.get("operationParams")
             nsd = db_nsr["nsd"]
             nsr_name = db_nsr["name"]   # TODO short-name??
 
@@ -600,43 +609,18 @@ class NsLcm(LcmBase):
                     db_vnfds_ref[vnfd_ref] = vnfd
                     db_vnfds[vnfd_id] = vnfd
 
-                # update VNFR vimAccount and pdu information
+            # Get or generates the _admin.deployed,VCA list
+            vca_deployed_list = None
+            if db_nsr["_admin"].get("deployed"):
+                vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
+            if vca_deployed_list is None:
+                vca_deployed_list = []
+                db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
+            elif isinstance(vca_deployed_list, dict):
+                # maintain backward compatibility. Change a dict to list at database
+                vca_deployed_list = list(vca_deployed_list.values())
+                db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 
-                vim_account = vnfr.get("vim-account-id")
-                vnfr_update = {}
-                if not vim_account:
-                    step = "Updating VNFR vimAccount"
-                    vim_account = db_nsr["instantiate_params"]["vimAccountId"]
-                    # check instantiate parameters
-                    if db_nsr["instantiate_params"].get("vnf"):
-                        for vnf_params in db_nsr["instantiate_params"]["vnf"]:
-                            if vnf_params.get("member-vnf-index") == vnfr["member-vnf-index-ref"]:
-                                if vnf_params.get("vimAccountId"):
-                                    vim_account = vnf_params.get("vimAccountId")
-                                break
-                    vnfr_update["vim-account-id"] = vim_account
-                    vnfr["vim-account-id"] = vim_account
-
-                # check PDUs and get PDU
-                for vdur_index, vdur in enumerate(get_iterable(vnfr, "vdur")):
-                    pdu_type = vdur.get("pdu-type")
-                    if not pdu_type:
-                        continue
-                    # look for free pdu
-                    self._look_for_pdu(vdur, vnfr, vim_account)
-                    # fill vnfr with pdu info
-                    vnfr_update["vdur.{}".format(vdur_index)] = vdur
-                if vnfr_update:
-                    self.update_db_2("vnfrs", vnfr["_id"], vnfr_update)
-
-            nsr_lcm = db_nsr["_admin"].get("deployed")
-            if not nsr_lcm:
-                nsr_lcm = db_nsr["_admin"]["deployed"] = {
-                    "id": nsr_id,
-                    "RO": {"vnfd_id": {}, "nsd_id": None, "nsr_id": None, "nsr_status": "SCHEDULED"},
-                    "nsr_ip": {},
-                    "VCA": {},
-                }
             db_nsr_update["detailed-status"] = "creating"
             db_nsr_update["operational-status"] = "init"
 
@@ -684,7 +668,7 @@ class NsLcm(LcmBase):
                 nsd_RO["id"] = RO_osm_nsd_id
                 nsd_RO.pop("_id", None)
                 nsd_RO.pop("_admin", None)
-                for c_vnf in nsd_RO["constituent-vnfd"]:
+                for c_vnf in nsd_RO.get("constituent-vnfd", ()):
                     vnfd_id = c_vnf["vnfd-id-ref"]
                     c_vnf["vnfd-id-ref"] = descriptor_id_2_RO[vnfd_id]
                 desc = await RO.create("nsd", descriptor=nsd_RO)
@@ -754,8 +738,7 @@ class NsLcm(LcmBase):
             detailed_status_old = None
             self.logger.debug(logging_text + step)
 
-            deployment_timeout = 2 * 3600   # Two hours
-            while deployment_timeout > 0:
+            while time() <= start_deploy + self.total_deploy_timeout:
                 desc = await RO.show("ns", RO_nsr_id)
                 ns_status, ns_status_info = RO.check_ns_status(desc)
                 db_nsr_update["admin.deployed.RO.nsr_status"] = ns_status
@@ -764,25 +747,23 @@ class NsLcm(LcmBase):
                 elif ns_status == "BUILD":
                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
                 elif ns_status == "ACTIVE":
-                    step = detailed_status = "Waiting for management IP address reported by the VIM"
+                    step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
                     try:
-                        nsr_lcm["nsr_ip"] = RO.get_ns_vnf_info(desc)
+                        self.ns_update_vnfr(db_vnfrs, desc)
                         break
-                    except ROclient.ROClientException as e:
-                        if e.http_code != 409:  # IP address is not ready return code is 409 CONFLICT
-                            raise e
+                    except LcmExceptionNoMgmtIP:
+                        pass
                 else:
                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
                 if detailed_status != detailed_status_old:
                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
                 await asyncio.sleep(5, loop=self.loop)
-                deployment_timeout -= 5
-            if deployment_timeout <= 0:
+            else:   # total_deploy_timeout
                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
 
-            step = "Updating VNFRs"
-            self.ns_update_vnfr(db_vnfrs, desc)
+            step = "Updating NSR"
+            self.ns_update_nsr(db_nsr_update, db_nsr, desc)
 
             db_nsr["detailed-status"] = "Configuring vnfr"
             self.update_db_2("nsrs", nsr_id, db_nsr_update)
@@ -790,7 +771,8 @@ class NsLcm(LcmBase):
             # The parameters we'll need to deploy a charm
             number_to_configure = 0
 
-            def deploy(vnf_index, vdu_id, mgmt_ip_address, n2vc_info, config_primitive=None):
+            def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, mgmt_ip_address, n2vc_info,
+                             config_primitive=None):
                 """An inner function to deploy the charm from either vnf or vdu
                 vnf_index is mandatory. vdu_id can be None for a vnf configuration or the id for vdu configuration
                 """
@@ -823,18 +805,21 @@ class NsLcm(LcmBase):
 
                 # ns_name will be ignored in the current version of N2VC
                 # but will be implemented for the next point release.
-                model_name = 'default'
-                vdu_id_text = "vnfd"
+                model_name = 'default'   # TODO bug 581 : change to nsr_id
                 if vdu_id:
                     vdu_id_text = vdu_id
-                application_name = self.n2vc.FormatApplicationName(
-                    nsr_name,
-                    vnf_index,
-                    vdu_id_text
-                )
-                if not nsr_lcm.get("VCA"):
-                    nsr_lcm["VCA"] = {}
-                nsr_lcm["VCA"][application_name] = db_nsr_update["_admin.deployed.VCA.{}".format(application_name)] = {
+                else:
+                    vdu_id_text = "vnfd"  # TODO bug 581 remove and add just an empty string ""
+                application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index, vdu_id_text)
+                # TODO bug 581 Add "-" as a final argument
+
+                vca_index = len(vca_deployed_list)
+                # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
+                # 26*26 charm in the same NS
+                # TODO bug 581 uncoment
+                # application_name = application_name[0:48]
+                # application_name += chr(97 + vca_index / 26) + chr(97 + vca_index % 26)
+                vca_deployed_ = {
                     "member-vnf-index": vnf_index,
                     "vdu_id": vdu_id,
                     "model": model_name,
@@ -842,7 +827,11 @@ class NsLcm(LcmBase):
                     "operational-status": "init",
                     "detailed-status": "",
                     "vnfd_id": vnfd_id,
+                    "vdu_name": vdu_name,
+                    "vdu_count_index": vdu_count_index,
                 }
+                vca_deployed_list.append(vca_deployed_)
+                db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 
                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
@@ -852,7 +841,7 @@ class NsLcm(LcmBase):
                     n2vc_info["nslcmop_id"] = nslcmop_id
                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
                     n2vc_info["lcmOperationType"] = "instantiate"
-                    n2vc_info["deployed"] = nsr_lcm["VCA"]
+                    n2vc_info["deployed"] = vca_deployed_list
                     n2vc_info["db_update"] = db_nsr_update
                 task = asyncio.ensure_future(
                     self.n2vc.DeployCharms(
@@ -874,7 +863,7 @@ class NsLcm(LcmBase):
             step = "Looking for needed vnfd to configure"
             self.logger.debug(logging_text + step)
 
-            for c_vnf in nsd["constituent-vnfd"]:
+            for c_vnf in get_iterable(nsd, "constituent-vnfd"):
                 vnfd_id = c_vnf["vnfd-id-ref"]
                 vnf_index = str(c_vnf["member-vnf-index"])
                 vnfd = db_vnfds_ref[vnfd_id]
@@ -894,12 +883,12 @@ class NsLcm(LcmBase):
                         # subsequent calls will be a nop and return immediately.
                         step = "connecting to N2VC to configure vnf {}".format(vnf_index)
                         await self.n2vc.login()
-                        deploy(vnf_index, None, db_vnfrs[vnf_index]["ip-address"], n2vc_info, config_primitive)
+                        deploy_charm(vnf_index, None, None, None, db_vnfrs[vnf_index]["ip-address"], n2vc_info,
+                                     config_primitive)
                         number_to_configure += 1
 
                 # Deploy charms for each VDU that supports one.
-                vdu_index = 0
-                for vdu in vnfd['vdu']:
+                for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
                     vdu_config = vdu.get('vdu-configuration')
                     proxy_charm = None
                     config_primitive = None
@@ -913,10 +902,14 @@ class NsLcm(LcmBase):
                         if proxy_charm:
                             step = "connecting to N2VC to configure vdu {} from vnf {}".format(vdu["id"], vnf_index)
                             await self.n2vc.login()
-                            deploy(vnf_index, vdu["id"], db_vnfrs[vnf_index]["vdur"][vdu_index]["ip-address"],
-                                   n2vc_info, config_primitive)
+                            vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
+                            # TODO for the moment only first vdu_id contains a charm deployed
+                            if vdur["vdu-id-ref"] != vdu["id"]:
+                                raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
+                                                   .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
+                            deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
+                                         vdur["ip-address"], n2vc_info, config_primitive)
                             number_to_configure += 1
-                    vdu_index += 1
 
             db_nsr_update["operational-status"] = "running"
             configuration_failed = False
@@ -927,30 +920,43 @@ class NsLcm(LcmBase):
                 db_nslcmop_update["detailed-status"] = old_status
 
                 # wait until all are configured.
-                while True:
+                while time() <= start_deploy + self.total_deploy_timeout:
                     if db_nsr_update:
                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
                     if db_nslcmop_update:
                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
+                    # TODO add a fake tast that set n2vc_event after some time
                     await n2vc_info["n2vc_event"].wait()
                     n2vc_info["n2vc_event"].clear()
                     all_active = True
                     status_map = {}
                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
-                    for _, vca_info in nsr_lcm["VCA"].items():
-                        vca_status = vca_info["operational-status"]
+                    now = time()
+                    for vca_deployed in vca_deployed_list:
+                        vca_status = vca_deployed["operational-status"]
                         if vca_status not in status_map:
                             # Initialize it
                             status_map[vca_status] = 0
                         status_map[vca_status] += 1
 
-                        if vca_status != "active":
-                            all_active = False
+                        if vca_status == "active":
+                            vca_deployed.pop("time_first_error", None)
+                            vca_deployed.pop("status_first_error", None)
+                            continue
+
+                        all_active = False
                         if vca_status in ("error", "blocked"):
-                            n2vc_error_text.append(
-                                "member_vnf_index={} vdu_id={} {}: {}".format(vca_info["member-vnf-index"],
-                                                                              vca_info["vdu_id"], vca_status,
-                                                                              vca_info["detailed-status"]))
+                            vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
+                            # if not first time in this status error
+                            if not vca_deployed.get("time_first_error"):
+                                vca_deployed["time_first_error"] = now
+                                continue
+                        if vca_deployed.get("time_first_error") and \
+                                now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
+                            n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
+                                                   .format(vca_deployed["member-vnf-index"],
+                                                           vca_deployed["vdu_id"], vca_status,
+                                                           vca_deployed["detailed-status-error"]))
 
                     if all_active:
                         break
@@ -974,6 +980,8 @@ class NsLcm(LcmBase):
                             db_nsr_update["detailed-status"] = cs
                             db_nslcmop_update["detailed-status"] = cs
                             old_status = cs
+                else:   # total_deploy_timeout
+                    raise LcmException("Timeout waiting ns to be configured")
 
             if not configuration_failed:
                 # all is done
@@ -1028,6 +1036,7 @@ class NsLcm(LcmBase):
         failed_detail = []   # annotates all failed error messages
         vca_task_list = []
         vca_task_dict = {}
+        vca_application_name2index = {}
         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
         db_nslcmop_update = {}
         nslcmop_operation_state = None
@@ -1037,7 +1046,7 @@ class NsLcm(LcmBase):
             step = "Getting nsr={} from db".format(nsr_id)
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
             # nsd = db_nsr["nsd"]
-            nsr_lcm = deepcopy(db_nsr["_admin"].get("deployed"))
+            nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
                 return
             # TODO ALF remove
@@ -1048,29 +1057,37 @@ class NsLcm(LcmBase):
             db_nsr_update["operational-status"] = "terminating"
             db_nsr_update["config-status"] = "terminating"
 
-            if nsr_lcm and nsr_lcm.get("VCA"):
+            if nsr_deployed and nsr_deployed.get("VCA"):
                 try:
                     step = "Scheduling configuration charms removing"
                     db_nsr_update["detailed-status"] = "Deleting charms"
                     self.logger.debug(logging_text + step)
                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
-                    for application_name, deploy_info in nsr_lcm["VCA"].items():
-                        if deploy_info:  # TODO it would be desirable having a and deploy_info.get("deployed"):
+                    # for backward compatibility
+                    if isinstance(nsr_deployed["VCA"], dict):
+                        nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
+                        db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
+                        self.update_db_2("nsrs", nsr_id, db_nsr_update)
+
+                    for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
+                        if vca_deployed:  # TODO it would be desirable having a and deploy_info.get("deployed"):
                             task = asyncio.ensure_future(
                                 self.n2vc.RemoveCharms(
-                                    deploy_info['model'],
-                                    application_name,
+                                    vca_deployed['model'],
+                                    vca_deployed["application"],
                                     # self.n2vc_callback,
                                     # db_nsr,
                                     # db_nslcmop,
                                 )
                             )
+                            vca_application_name2index[vca_deployed["application"]] = vca_index
                             vca_task_list.append(task)
-                            vca_task_dict[application_name] = task
-                            # task.add_done_callback(functools.partial(self.n2vc_callback, deploy_info['model'],
-                            #                                          deploy_info['application'], None, db_nsr,
+                            vca_task_dict[vca_deployed["application"]] = task
+                            # task.add_done_callback(functools.partial(self.n2vc_callback, vca_deployed['model'],
+                            #                                          vca_deployed['application'], None, db_nsr,
                             #                                          db_nslcmop, vnf_index))
-                            self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "delete_charm:" + application_name, task)
+                            self.lcm_tasks.register("ns", nsr_id, nslcmop_id,
+                                                    "delete_charm:" + vca_deployed["application"], task)
                 except Exception as e:
                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
 
@@ -1080,9 +1097,9 @@ class NsLcm(LcmBase):
 
             # Delete ns
             RO_nsr_id = RO_delete_action = None
-            if nsr_lcm and nsr_lcm.get("RO"):
-                RO_nsr_id = nsr_lcm["RO"].get("nsr_id")
-                RO_delete_action = nsr_lcm["RO"].get("nsr_delete_action_id")
+            if nsr_deployed and nsr_deployed.get("RO"):
+                RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
+                RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
             try:
                 if RO_nsr_id:
                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = "Deleting ns at RO"
@@ -1134,8 +1151,8 @@ class NsLcm(LcmBase):
                     RO_fail = True
 
             # Delete nsd
-            if not RO_fail and nsr_lcm and nsr_lcm.get("RO") and nsr_lcm["RO"].get("nsd_id"):
-                RO_nsd_id = nsr_lcm["RO"]["nsd_id"]
+            if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
+                RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
                 try:
                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
                         "Deleting nsd at RO"
@@ -1155,8 +1172,8 @@ class NsLcm(LcmBase):
                         self.logger.error(logging_text + failed_detail[-1])
                         RO_fail = True
 
-            if not RO_fail and nsr_lcm and nsr_lcm.get("RO") and nsr_lcm["RO"].get("vnfd_id"):
-                for vnf_id, RO_vnfd_id in nsr_lcm["RO"]["vnfd_id"].items():
+            if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd_id"):
+                for vnf_id, RO_vnfd_id in nsr_deployed["RO"]["vnfd_id"].items():
                     if not RO_vnfd_id:
                         continue
                     try:
@@ -1184,17 +1201,20 @@ class NsLcm(LcmBase):
                 await asyncio.wait(vca_task_list, timeout=300)
             for application_name, task in vca_task_dict.items():
                 if task.cancelled():
-                    failed_detail.append("VCA[{}] Deletion has been cancelled".format(application_name))
+                    failed_detail.append("VCA[application_name={}] Deletion has been cancelled"
+                                         .format(application_name))
                 elif task.done():
                     exc = task.exception()
                     if exc:
-                        failed_detail.append("VCA[{}] Deletion exception: {}".format(application_name, exc))
+                        failed_detail.append("VCA[application_name={}] Deletion exception: {}"
+                                             .format(application_name, exc))
                     else:
-                        db_nsr_update["_admin.deployed.VCA.{}".format(application_name)] = None
+                        vca_index = vca_application_name2index[application_name]
+                        db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
                 else:  # timeout
                     # TODO Should it be cancelled?!!
                     task.cancel()
-                    failed_detail.append("VCA[{}] Deletion timeout".format(application_name))
+                    failed_detail.append("VCA[application_name={}] Deletion timeout".format(application_name))
 
             if failed_detail:
                 self.logger.error(logging_text + " ;".join(failed_detail))
@@ -1249,27 +1269,30 @@ class NsLcm(LcmBase):
             self.logger.debug(logging_text + "Exit")
             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
 
-    async def _ns_execute_primitive(self, db_deployed, nsr_name, member_vnf_index, vdu_id, primitive, primitive_params):
+    async def _ns_execute_primitive(self, db_deployed, nsr_name, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
+                                    primitive, primitive_params):
 
-        vdu_id_text = "vnfd"
-        if vdu_id:
-            vdu_id_text = vdu_id
-        application_name = self.n2vc.FormatApplicationName(
-            nsr_name,
-            member_vnf_index,
-            vdu_id_text
-        )
-        vca_deployed = db_deployed["VCA"].get(application_name)
-        if not vca_deployed:
-            raise LcmException("charm for member_vnf_index={} vdu_id={} is not deployed".format(member_vnf_index,
-                                                                                                vdu_id))
+        for vca_deployed in db_deployed["VCA"]:
+            if not vca_deployed:
+                continue
+            if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
+                continue
+            if vdu_name and vdu_name != vca_deployed["vdu_name"]:
+                continue
+            if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
+                continue
+            break
+        else:
+            raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not deployed"
+                               .format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
         model_name = vca_deployed.get("model")
         application_name = vca_deployed.get("application")
         if not model_name or not application_name:
-            raise LcmException("charm for member_vnf_index={} is not properly deployed".format(member_vnf_index))
+            raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not model "
+                               "or application name" .format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
         if vca_deployed["operational-status"] != "active":
-            raise LcmException("charm for member_vnf_index={} operational_status={} not 'active'".format(
-                member_vnf_index, vca_deployed["operational-status"]))
+            raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
+                member_vnf_index, vdu_id, vca_deployed["operational-status"]))
         callback = None  # self.n2vc_callback
         callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
         await self.n2vc.login()
@@ -1320,10 +1343,12 @@ class NsLcm(LcmBase):
             step = "Getting information from database"
             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
-            nsr_lcm = db_nsr["_admin"].get("deployed")
+            nsr_deployed = db_nsr["_admin"].get("deployed")
             nsr_name = db_nsr["name"]
             vnf_index = db_nslcmop["operationParams"]["member_vnf_index"]
             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
+            vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
+            vdu_name = db_nslcmop["operationParams"].get("vdu_name")
 
             # look if previous tasks in process
             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
@@ -1336,10 +1361,17 @@ class NsLcm(LcmBase):
                 if pending:
                     raise LcmException("Timeout waiting related tasks to be completed")
 
+            # for backward compatibility
+            if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
+                nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
+                db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
+                self.update_db_2("nsrs", nsr_id, db_nsr_update)
+
             # TODO check if ns is in a proper status
             primitive = db_nslcmop["operationParams"]["primitive"]
             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
-            result, result_detail = await self._ns_execute_primitive(nsr_lcm, nsr_name, vnf_index, vdu_id, primitive,
+            result, result_detail = await self._ns_execute_primitive(nsr_deployed, nsr_name, vnf_index, vdu_id,
+                                                                     vdu_name, vdu_count_index, primitive,
                                                                      primitive_params)
             db_nslcmop_update["detailed-status"] = result_detail
             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
@@ -1390,11 +1422,13 @@ class NsLcm(LcmBase):
         scale_process = None
         old_operational_status = ""
         old_config_status = ""
+        vnfr_scaled = False
         try:
             step = "Getting nslcmop from database"
             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
             step = "Getting nsr from database"
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
+            nsr_name = db_nsr["name"]
             old_operational_status = db_nsr["operational-status"]
             old_config_status = db_nsr["config-status"]
 
@@ -1412,13 +1446,19 @@ class NsLcm(LcmBase):
             step = "Parsing scaling parameters"
             db_nsr_update["operational-status"] = "scaling"
             self.update_db_2("nsrs", nsr_id, db_nsr_update)
-            nsr_lcm = db_nsr["_admin"].get("deployed")
-            RO_nsr_id = nsr_lcm["RO"]["nsr_id"]
+            nsr_deployed = db_nsr["_admin"].get("deployed")
+            RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
 
+            # for backward compatibility
+            if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
+                nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
+                db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
+                self.update_db_2("nsrs", nsr_id, db_nsr_update)
+
             step = "Getting vnfr from database"
             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
             step = "Getting vnfd from database"
@@ -1529,8 +1569,9 @@ class NsLcm(LcmBase):
                                 "primitive".format(scaling_group, config_primitive))
                         scale_process = "VCA"
                         db_nsr_update["config-status"] = "configuring pre-scaling"
-                        result, result_detail = await self._ns_execute_primitive(nsr_lcm, vnf_index,
-                                                                                 vnf_config_primitive, primitive_params)
+                        result, result_detail = await self._ns_execute_primitive(nsr_deployed, nsr_name, vnf_index,
+                                                                                 None, None, None, vnf_config_primitive,
+                                                                                 primitive_params)
                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
                             vnf_config_primitive, result, result_detail))
                         if result == "FAILED":
@@ -1577,14 +1618,18 @@ class NsLcm(LcmBase):
                         elif ns_status == "BUILD":
                             detailed_status = step + "; {}".format(ns_status_info)
                         elif ns_status == "ACTIVE":
-                            step = detailed_status = "Waiting for management IP address reported by the VIM"
+                            step = detailed_status = \
+                                "Waiting for management IP address reported by the VIM. Updating VNFRs"
+                            if not vnfr_scaled:
+                                self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
+                                vnfr_scaled = True
                             try:
                                 desc = await RO.show("ns", RO_nsr_id)
-                                nsr_lcm["nsr_ip"] = RO.get_ns_vnf_info(desc)
+                                # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
+                                self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
                                 break
-                            except ROclient.ROClientException as e:
-                                if e.http_code != 409:  # IP address is not ready return code is 409 CONFLICT
-                                    raise e
+                            except LcmExceptionNoMgmtIP:
+                                pass
                         else:
                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
                     if detailed_status != detailed_status_old:
@@ -1596,10 +1641,6 @@ class NsLcm(LcmBase):
                 if deployment_timeout <= 0:
                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
 
-                step = "Updating VNFRs"
-                self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
-                self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
-
                 # update VDU_SCALING_INFO with the obtained ip_addresses
                 if vdu_scaling_info["scaling_direction"] == "OUT":
                     for vdur in reversed(db_vnfr["vdur"]):
@@ -1650,8 +1691,9 @@ class NsLcm(LcmBase):
                         scale_process = "VCA"
                         db_nsr_update["config-status"] = "configuring post-scaling"
 
-                        result, result_detail = await self._ns_execute_primitive(nsr_lcm, vnf_index,
-                                                                                 vnf_config_primitive, primitive_params)
+                        result, result_detail = await self._ns_execute_primitive(nsr_deployed, nsr_name, vnf_index,
+                                                                                 None, None, None, vnf_config_primitive,
+                                                                                 primitive_params)
                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
                             vnf_config_primitive, result, result_detail))
                         if result == "FAILED":