Add related vim_wim_tasks (tasks that deals with same vim/wim element 77/7277/9
authortierno <alfonso.tiernosepulveda@telefonica.com>
Mon, 4 Mar 2019 17:32:01 +0000 (17:32 +0000)
committertierno <alfonso.tiernosepulveda@telefonica.com>
Mon, 15 Apr 2019 07:18:14 +0000 (08:18 +0100)
Remove state of vim_threads, working with database instead of memory
Allow several vim_threads over the same vim tasks

Change-Id: I5825b69309b8d3e6146798747b002108a34fa2aa
Signed-off-by: tierno <alfonso.tiernosepulveda@telefonica.com>
database_utils/migrate_mano_db.sh
openmanod
osm_ro/db_base.py
osm_ro/nfvo.py
osm_ro/nfvo_db.py
osm_ro/openmano_schemas.py
osm_ro/vim_thread.py
osm_ro/wim/engine.py

index b6867ff..b587612 100755 (executable)
@@ -36,7 +36,7 @@ QUIET_MODE=""
 BACKUP_DIR=""
 BACKUP_FILE=""
 #TODO update it with the last database version
-LAST_DB_VERSION=37
+LAST_DB_VERSION=38
 
 # Detect paths
 MYSQL=$(which mysql)
@@ -195,6 +195,7 @@ fi
 #[ $OPENMANO_VER_NUM -ge 6001 ] && DB_VERSION=35  #0.6.01 =>  35
 #[ $OPENMANO_VER_NUM -ge 6003 ] && DB_VERSION=36  #0.6.03 =>  36
 #[ $OPENMANO_VER_NUM -ge 6009 ] && DB_VERSION=37  #0.6.09 =>  37
+#[ $OPENMANO_VER_NUM -ge 6011 ] && DB_VERSION=38  #0.6.11 =>  38
 #TODO ... put next versions here
 
 function upgrade_to_1(){
@@ -1362,6 +1363,49 @@ function downgrade_from_37(){
     # It doesn't make sense to reverse to a bug state.
     sql "DELETE FROM schema_version WHERE version_int='37';"
 }
+function upgrade_to_38(){
+    echo "      Change vim_wim_actions, add worker, related"
+    sql "ALTER TABLE vim_wim_actions ADD COLUMN worker VARCHAR(64) NULL AFTER task_index, " \
+           "ADD COLUMN related VARCHAR(36) NULL AFTER worker, " \
+           "CHANGE COLUMN status status ENUM('SCHEDULED','BUILD','DONE','FAILED','SUPERSEDED','FINISHED') " \
+           "NOT NULL DEFAULT 'SCHEDULED' AFTER item_id;"
+       sql "UPDATE vim_wim_actions set related=item_id;"
+       echo "      Change DONE to FINISHED when DELETE has been completed"
+       sql "UPDATE vim_wim_actions as v1 join vim_wim_actions as v2 on (v1.action='CREATE' or v1.action='FIND') and " \
+           "v2.action='DELETE' and (v2.status='SUPERSEDED' or v2.status='DONE') and v1.item_id=v2.item_id " \
+        "SET v1.status='FINISHED', v2.status='FINISHED';"
+    echo "      Add osm_id to instance_nets"
+    sql "ALTER TABLE instance_nets ADD COLUMN osm_id VARCHAR(255) NULL AFTER uuid;"
+    echo "      Add related to instance_xxxx"
+    for table in instance_classifications instance_nets instance_sfis instance_sfps instance_sfs \
+        instance_vms
+    do
+        sql "ALTER TABLE $table ADD COLUMN related VARCHAR(36) NULL AFTER vim_info;"
+       sql "UPDATE $table set related=uuid;"
+    done
+    sql "ALTER TABLE instance_wim_nets ADD COLUMN related VARCHAR(36) NULL AFTER wim_info;"
+       sql "UPDATE instance_wim_nets set related=uuid;"
+
+    sql "INSERT INTO schema_version (version_int, version, openmano_ver, comments, date) " \
+        "VALUES (38, '0.38', '0.6.11', 'Adding related to vim_wim_actions', '2019-03-07');"
+
+}
+function downgrade_from_38(){
+    echo "      Change vim_wim_actions, delete worker, related"
+       sql "UPDATE vim_wim_actions SET status='DONE' WHERE status='FINISHED';"
+    sql "ALTER TABLE vim_wim_actions DROP COLUMN worker, DROP COLUMN related, " \
+           "CHANGE COLUMN status status ENUM('SCHEDULED','BUILD','DONE','FAILED','SUPERSEDED') " \
+           "NOT NULL DEFAULT 'SCHEDULED' AFTER item_id;"
+    echo "      Remove related from instance_xxxx"
+    for table in instance_classifications instance_nets instance_wim_netsinstance_sfis instance_sfps instance_sfs \
+        instance_vms
+    do
+        sql "ALTER TABLE $table DROP COLUMN related;"
+    done
+    echo "      Remove osm_id from instance_nets"
+    sql "ALTER TABLE instance_nets DROP COLUMN osm_id;"
+    sql "DELETE FROM schema_version WHERE version_int='38';"
+}
 
 #TODO ... put functions here
 
index 6e46990..07d2a8e 100755 (executable)
--- a/openmanod
+++ b/openmanod
@@ -50,9 +50,9 @@ import osm_ro
 
 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes"
 __date__ = "$26-aug-2014 11:09:29$"
-__version__ = "0.6.10"
+__version__ = "0.6.11"
 version_date = "Mar 2019"
-database_version = 37      # expected database schema version
+database_version = 38      # expected database schema version
 
 global global_config
 global logger
index e946f00..be681e3 100644 (file)
@@ -562,7 +562,7 @@ class db_base():
             INSERT: dictionary with the key:value to insert
             table: table where to insert
             add_uuid: if True, it will create an uuid key entry at INSERT if not provided
-            created_time: time to add to the created_time column
+            created_time: time to add to the created_at column
         It checks presence of uuid and add one automatically otherwise
         Return: uuid
         '''
@@ -591,7 +591,7 @@ class db_base():
         cmd= "INSERT INTO " + table +" SET " + \
             ",".join(map(self.__tuple2db_format_set, INSERT.iteritems() ))
         if created_time:
-            cmd += ",created_at=%f" % created_time
+            cmd += ",created_at={time},modified_at={time}".format(time=created_time)
         if confidential_data:
             index = cmd.find("SET")
             subcmd = cmd[:index] + 'SET...'
@@ -627,7 +627,7 @@ class db_base():
 
     @retry
     @with_transaction
-    def update_rows(self, table, UPDATE, WHERE, modified_time=0, attempt=_ATTEMPT):
+    def update_rows(self, table, UPDATE, WHERE, modified_time=None, attempt=_ATTEMPT):
         """ Update one or several rows of a table.
         :param UPDATE: dictionary with the changes. dict keys are database columns that will be set with the dict values
         :param table: database table to update
@@ -638,11 +638,12 @@ class db_base():
                 keys can be suffixed by >,<,<>,>=,<= so that this is used to compare key and value instead of "="
                 The special keys "OR", "AND" with a dict value is used to create a nested WHERE
             If a list, each item will be a dictionary that will be concatenated with OR
-        :param modified_time: Can contain the time to be set to the table row
+        :param modified_time: Can contain the time to be set to the table row.
+            None to set automatically, 0 to do not modify it
         :return: the number of updated rows, raises exception upon error
         """
-        if table in self.tables_with_created_field and modified_time==0:
-            modified_time=time.time()
+        if table in self.tables_with_created_field and modified_time is None:
+            modified_time = time.time()
 
         return self._update_rows(table, UPDATE, WHERE, modified_time)
 
index ff47fec..d433ac9 100644 (file)
@@ -305,6 +305,9 @@ def clean_db(mydb):
         nb_deleted += len(actions_to_delete)
         if len(actions_to_delete) < 100:
             break
+    # clean locks
+    mydb.update_rows("vim_wim_actions", UPDATE={"worker": None}, WHERE={"worker<>": None})
+
     if nb_deleted:
         logger.debug("Removed {} unused vim_wim_actions".format(nb_deleted))
 
@@ -3224,8 +3227,16 @@ def create_instance(mydb, tenant_id, instance_dict):
             # <-- WIM
 
             descriptor_net = {}
-            if instance_dict.get("networks") and instance_dict["networks"].get(sce_net["name"]):
-                descriptor_net = instance_dict["networks"][sce_net["name"]]
+            if instance_dict.get("networks"):
+                if sce_net.get("uuid") in instance_dict["networks"]:
+                    descriptor_net = instance_dict["networks"][sce_net["uuid"]]
+                    descriptor_net_name = sce_net["uuid"]
+                elif sce_net.get("osm_id") in instance_dict["networks"]:
+                    descriptor_net = instance_dict["networks"][sce_net["osm_id"]]
+                    descriptor_net_name = sce_net["osm_id"]
+                elif sce_net["name"] in instance_dict["networks"]:
+                    descriptor_net = instance_dict["networks"][sce_net["name"]]
+                    descriptor_net_name = sce_net["name"]
             net_name = descriptor_net.get("vim-network-name")
             # add datacenters from instantiation parameters
             if descriptor_net.get("sites"):
@@ -3235,6 +3246,22 @@ def create_instance(mydb, tenant_id, instance_dict):
             sce_net2instance[sce_net_uuid] = {}
             net2task_id['scenario'][sce_net_uuid] = {}
 
+            use_network = None
+            related_network = None
+            if descriptor_net.get("use-network"):
+                target_instance_nets = mydb.get_rows(
+                    SELECT="related",
+                    FROM="instance_nets",
+                    WHERE={"instance_scenario_id": descriptor_net["use-network"]["instance_scenario_id"],
+                           "osm_id":  descriptor_net["use-network"]["osm_id"]},
+                )
+                if not target_instance_nets:
+                    raise NfvoException(
+                        "Cannot find the target network at instance:networks[{}]:use-network".format(descriptor_net_name),
+                        httperrors.Bad_Request)
+                else:
+                    use_network = target_instance_nets[0]["related"]
+
             if sce_net["external"]:
                 number_mgmt_networks += 1
 
@@ -3322,8 +3349,12 @@ def create_instance(mydb, tenant_id, instance_dict):
                 net_uuid = str(uuid4())
                 uuid_list.append(net_uuid)
                 sce_net2instance[sce_net_uuid][datacenter_id] = net_uuid
+                if not related_network:   # all db_instance_nets will have same related
+                    related_network = use_network or net_uuid
                 db_net = {
                     "uuid": net_uuid,
+                    "osm_id": sce_net.get("osm_id") or sce_net["name"],
+                    "related": related_network,
                     'vim_net_id': None,
                     "vim_name": net_vim_name,
                     "instance_scenario_id": instance_uuid,
@@ -3342,6 +3373,7 @@ def create_instance(mydb, tenant_id, instance_dict):
                     "action": task_action,
                     "item": "instance_nets",
                     "item_id": net_uuid,
+                    "related": related_network,
                     "extra": yaml.safe_dump(task_extra, default_flow_style=True, width=256)
                 }
                 net2task_id['scenario'][sce_net_uuid][datacenter_id] = task_index
@@ -3422,6 +3454,7 @@ def create_instance(mydb, tenant_id, instance_dict):
                         uuid_list.append(sfi_uuid)
                         db_sfi = {
                             "uuid": sfi_uuid,
+                            "related": sfi_uuid,
                             "instance_scenario_id": instance_uuid,
                             'sce_rsp_hop_id': cp['uuid'],
                             'datacenter_id': datacenter_id,
@@ -3437,6 +3470,7 @@ def create_instance(mydb, tenant_id, instance_dict):
                             "status": "SCHEDULED",
                             "item": "instance_sfis",
                             "item_id": sfi_uuid,
+                            "related": sfi_uuid,
                             "extra": yaml.safe_dump({"params": extra_params, "depends_on": [dependencies[i]]},
                                                     default_flow_style=True, width=256)
                         }
@@ -3448,6 +3482,7 @@ def create_instance(mydb, tenant_id, instance_dict):
                     uuid_list.append(sf_uuid)
                     db_sf = {
                         "uuid": sf_uuid,
+                        "related": sf_uuid,
                         "instance_scenario_id": instance_uuid,
                         'sce_rsp_hop_id': cp['uuid'],
                         'datacenter_id': datacenter_id,
@@ -3463,6 +3498,7 @@ def create_instance(mydb, tenant_id, instance_dict):
                         "status": "SCHEDULED",
                         "item": "instance_sfs",
                         "item_id": sf_uuid,
+                        "related": sf_uuid,
                         "extra": yaml.safe_dump({"params": "", "depends_on": sfis_created},
                                                 default_flow_style=True, width=256)
                     }
@@ -3493,6 +3529,7 @@ def create_instance(mydb, tenant_id, instance_dict):
                         uuid_list.append(classification_uuid)
                         db_classification = {
                             "uuid": classification_uuid,
+                            "related": classification_uuid,
                             "instance_scenario_id": instance_uuid,
                             'sce_classifier_match_id': match['uuid'],
                             'datacenter_id': datacenter_id,
@@ -3515,6 +3552,7 @@ def create_instance(mydb, tenant_id, instance_dict):
                             "status": "SCHEDULED",
                             "item": "instance_classifications",
                             "item_id": classification_uuid,
+                            "related": classification_uuid,
                             "extra": yaml.safe_dump({"params": classification_params, "depends_on": [dependencies[i]]},
                                                     default_flow_style=True, width=256)
                         }
@@ -3527,6 +3565,7 @@ def create_instance(mydb, tenant_id, instance_dict):
                 uuid_list.append(sfp_uuid)
                 db_sfp = {
                     "uuid": sfp_uuid,
+                    "related": sfp_uuid,
                     "instance_scenario_id": instance_uuid,
                     'sce_rsp_id': rsp['uuid'],
                     'datacenter_id': datacenter_id,
@@ -3542,6 +3581,7 @@ def create_instance(mydb, tenant_id, instance_dict):
                     "status": "SCHEDULED",
                     "item": "instance_sfps",
                     "item_id": sfp_uuid,
+                    "related": sfp_uuid,
                     "extra": yaml.safe_dump({"params": "", "depends_on": sfs_created + classifications_created},
                                             default_flow_style=True, width=256)
                 }
@@ -3657,6 +3697,7 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList):
         vnf_net2instance[sce_vnf['uuid']][net['uuid']] = net_uuid
         db_net = {
             "uuid": net_uuid,
+            "related": net_uuid,
             'vim_net_id': None,
             "vim_name": net_name,
             "instance_scenario_id": instance_uuid,
@@ -3687,6 +3728,7 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList):
             "action": task_action,
             "item": "instance_nets",
             "item_id": net_uuid,
+            "related": net_uuid,
             "extra": yaml.safe_dump(task_extra, default_flow_style=True, width=256)
         }
         task_index += 1
@@ -3926,6 +3968,7 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList):
             uuid_list.append(vm_uuid)
             db_vm = {
                 "uuid": vm_uuid,
+                "related": vm_uuid,
                 'instance_vnf_id': vnf_uuid,
                 # TODO delete "vim_vm_id": vm_id,
                 "vm_id": vm["uuid"],
@@ -3965,6 +4008,7 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList):
                 "status": "SCHEDULED",
                 "item": "instance_vms",
                 "item_id": vm_uuid,
+                "related": vm_uuid,
                 "extra": yaml.safe_dump({"params": task_params, "depends_on": task_depends_on},
                                         default_flow_style=True, width=256)
             }
@@ -4042,6 +4086,7 @@ def delete_instance(mydb, tenant_id, instance_id):
             "status": "SCHEDULED",
             "item": "instance_sfps",
             "item_id": sfp["uuid"],
+            "related": sfp["related"],
             "extra": yaml.safe_dump(extra, default_flow_style=True, width=256)
         }
         task_index += 1
@@ -4082,6 +4127,7 @@ def delete_instance(mydb, tenant_id, instance_id):
             "status": "SCHEDULED",
             "item": "instance_classifications",
             "item_id": classification["uuid"],
+            "related": classification["related"],
             "extra": yaml.safe_dump(extra, default_flow_style=True, width=256)
         }
         task_index += 1
@@ -4120,6 +4166,7 @@ def delete_instance(mydb, tenant_id, instance_id):
             "status": "SCHEDULED",
             "item": "instance_sfs",
             "item_id": sf["uuid"],
+            "related": sf["related"],
             "extra": yaml.safe_dump(extra, default_flow_style=True, width=256)
         }
         task_index += 1
@@ -4158,6 +4205,7 @@ def delete_instance(mydb, tenant_id, instance_id):
             "status": "SCHEDULED",
             "item": "instance_sfis",
             "item_id": sfi["uuid"],
+            "related": sfi["related"],
             "extra": yaml.safe_dump(extra, default_flow_style=True, width=256)
         }
         task_index += 1
@@ -4199,6 +4247,7 @@ def delete_instance(mydb, tenant_id, instance_id):
                 "status": "SCHEDULED",
                 "item": "instance_vms",
                 "item_id": vm["uuid"],
+                "related": vm["related"],
                 "extra": yaml.safe_dump({"params": vm["interfaces"], "depends_on": sfi_dependencies},
                                         default_flow_style=True, width=256)
             }
@@ -4253,6 +4302,7 @@ def delete_instance(mydb, tenant_id, instance_id):
             "status": "SCHEDULED",
             "item": "instance_nets",
             "item_id": net["uuid"],
+            "related": net["related"],
             "extra": yaml.safe_dump(extra, default_flow_style=True, width=256)
         }
         task_index += 1
@@ -4552,6 +4602,7 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict):
                         "status": "SCHEDULED",
                         "item": "instance_vms",
                         "item_id": vdu_id,
+                        "related": vm["related"],
                         "extra": yaml.safe_dump({"params": vm_interfaces},
                                                 default_flow_style=True, width=256)
                     }
@@ -4642,6 +4693,7 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict):
                         "status": "SCHEDULED",
                         "item": "instance_vms",
                         "item_id": vm_uuid,
+                        "related": target_vm["related"],
                         # ALF
                         # ALF
                         # TODO examinar parametros, quitar MAC o incrementar. Incrementar IP y colocar las dependencias con ACTION-asdfasd.
index 86144de..eb72b13 100644 (file)
@@ -964,7 +964,7 @@ class nfvo_db(db_base.db_base):
 
             # instance vms
             cmd = "SELECT iv.uuid as uuid, vim_vm_id, status, error_msg, vim_info, iv.created_at as "\
-                    "created_at, name, vms.osm_id as vdu_osm_id, vim_name, vms.uuid as vm_uuid"\
+                    "created_at, name, vms.osm_id as vdu_osm_id, vim_name, vms.uuid as vm_uuid, related"\
                     " FROM instance_vms as iv join vms on iv.vm_id=vms.uuid "\
                     " WHERE instance_vnf_id='{}' ORDER BY iv.created_at".format(vnf['uuid'])
             self.logger.debug(cmd)
@@ -1002,7 +1002,7 @@ class nfvo_db(db_base.db_base):
         #where_text = "instance_nets.instance_scenario_id='"+ instance_dict['uuid'] + "'"
         cmd = "SELECT inets.uuid as uuid,vim_net_id,status,error_msg,vim_info,created, sce_net_id, " \
                 "net_id as vnf_net_id, datacenter_id, datacenter_tenant_id, sdn_net_id, " \
-                "snets.osm_id as ns_net_osm_id, nets.osm_id as vnf_net_osm_id, inets.vim_name " \
+                "snets.osm_id as ns_net_osm_id, nets.osm_id as vnf_net_osm_id, inets.vim_name, related " \
                 "FROM instance_nets as inets left join sce_nets as snets on inets.sce_net_id=snets.uuid " \
                 "left join nets on inets.net_id=nets.uuid " \
                 "WHERE instance_scenario_id='{}' ORDER BY inets.created_at".format(instance_dict['uuid'])
@@ -1012,7 +1012,7 @@ class nfvo_db(db_base.db_base):
 
         #instance_sfps
         cmd = "SELECT uuid,vim_sfp_id,sce_rsp_id,datacenter_id,"\
-                "datacenter_tenant_id,status,error_msg,vim_info"\
+                "datacenter_tenant_id,status,error_msg,vim_info, related"\
                 " FROM instance_sfps" \
                 " WHERE instance_scenario_id='{}' ORDER BY created_at".format(instance_dict['uuid'])
         self.logger.debug(cmd)
@@ -1022,7 +1022,7 @@ class nfvo_db(db_base.db_base):
         # for sfp in instance_dict['sfps']:
         #instance_sfs
         cmd = "SELECT uuid,vim_sf_id,sce_rsp_hop_id,datacenter_id,"\
-                "datacenter_tenant_id,status,error_msg,vim_info"\
+                "datacenter_tenant_id,status,error_msg,vim_info, related"\
                 " FROM instance_sfs" \
                 " WHERE instance_scenario_id='{}' ORDER BY created_at".format(instance_dict['uuid']) # TODO: replace instance_scenario_id with instance_sfp_id
         self.logger.debug(cmd)
@@ -1032,7 +1032,7 @@ class nfvo_db(db_base.db_base):
         #for sf in instance_dict['sfs']:
         #instance_sfis
         cmd = "SELECT uuid,vim_sfi_id,sce_rsp_hop_id,datacenter_id,"\
-                "datacenter_tenant_id,status,error_msg,vim_info"\
+                "datacenter_tenant_id,status,error_msg,vim_info, related"\
                 " FROM instance_sfis" \
                 " WHERE instance_scenario_id='{}' ORDER BY created_at".format(instance_dict['uuid']) # TODO: replace instance_scenario_id with instance_sf_id
         self.logger.debug(cmd)
@@ -1042,7 +1042,7 @@ class nfvo_db(db_base.db_base):
 
         #instance_classifications
         cmd = "SELECT uuid,vim_classification_id,sce_classifier_match_id,datacenter_id,"\
-                "datacenter_tenant_id,status,error_msg,vim_info"\
+                "datacenter_tenant_id,status,error_msg,vim_info, related"\
                 " FROM instance_classifications" \
                 " WHERE instance_scenario_id='{}' ORDER BY created_at".format(instance_dict['uuid'])
         self.logger.debug(cmd)
index 5839e24..988b6ca 100644 (file)
@@ -1094,6 +1094,16 @@ instance_scenario_create_schema_v01 = {
                                 },
                                 "wim_account": {"oneOf": [boolean_schema, id_schema, null_schema]},
                                 "ip-profile": ip_profile_schema,
+                                "use-network": {
+                                    "type": "object",
+                                    "properties": {
+                                        "instance_scenario_id": id_schema,
+                                        # "member_vnf_index": name_schema,  # if not null, network inside VNF
+                                        "osm_id": name_schema,  # sce_network osm_id or name
+                                    },
+                                    "additionalProperties": False,
+                                    "required": ["instance_scenario_id", "osm_id"]
+                                },
                                 #if the network connects VNFs deployed at different sites, you must specify one entry per site that this network connect to
                                 "sites": {
                                     "type":"array",
index 02b3bcc..8ed570c 100644 (file)
 """"
 This is thread that interacts with a VIM. It processes TASKs sequentially against a single VIM.
 The tasks are stored at database in table vim_wim_actions
+Several vim_wim_actions can refer to the same element at VIM (flavor, network, ...). This is somethng to avoid if RO
+is migrated to a non-relational database as mongo db. Each vim_wim_actions reference a different instance_Xxxxx
+In this case "related" colunm contains the same value, to know they refer to the same vim. In case of deletion, it
+there is related tasks using this element, it is not deleted, The vim_info needed to delete is transfered to other task
+
 The task content is (M: stored at memory, D: stored at database):
     MD  instance_action_id:  reference a global action over an instance-scenario: database instance_actions
     MD  task_index:     index number of the task. This together with the previous forms a unique key identifier
     MD  datacenter_vim_id:  should contain the uuid of the VIM managed by this thread
     MD  vim_id:     id of the vm,net,etc at VIM
-    MD  action:     CREATE, DELETE, FIND
     MD  item:       database table name, can be instance_vms, instance_nets, TODO: datacenter_flavors, datacenter_images
     MD  item_id:    uuid of the referenced entry in the previous table
-    MD  status:     SCHEDULED,BUILD,DONE,FAILED,SUPERSEDED
+    MD  action:     CREATE, DELETE, FIND
+    MD  status:     SCHEDULED: action need to be done
+                    BUILD: not used
+                    DONE: Done and it must be polled to VIM periodically to see status. ONLY for action=CREATE or FIND
+                    FAILED: It cannot be created/found/deleted
+                    FINISHED: similar to DONE, but no refresh is needed anymore. Task is maintained at database but
+                        it is never processed by any thread
+                    SUPERSEDED: similar to FINSISHED, but nothing has been done to completed the task.
     MD  extra:      text with yaml format at database, dict at memory with:
-            params:     list with the params to be sent to the VIM for CREATE or FIND. For DELETE the vim_id is taken from other related tasks
-            find:       (only for CREATE tasks) if present it should FIND before creating and use if existing. Contains the FIND params
-            depends_on: list with the 'task_index'es of tasks that must be completed before. e.g. a vm creation depends on a net creation
+            params:     list with the params to be sent to the VIM for CREATE or FIND. For DELETE the vim_id is taken
+                        from other related tasks
+            find:       (only for CREATE tasks) if present it should FIND before creating and use if existing. Contains
+                        the FIND params
+            depends_on: list with the 'task_index'es of tasks that must be completed before. e.g. a vm creation depends
+                        on a net creation
                         can contain an int (single index on the same instance-action) or str (compete action ID)
             sdn_net_id: used for net.
-            tries:
             interfaces: used for VMs. Each key is the uuid of the instance_interfaces entry at database
                 iface_id: uuid of intance_interfaces
                 sdn_port_id:
                 sdn_net_id:
+                vim_info
             created_items: dictionary with extra elements created that need to be deleted. e.g. ports, volumes,...
             created:    False if the VIM element is not created by other actions, and it should not be deleted
             vim_status: VIM status of the element. Stored also at database in the instance_XXX
-    M   depends:    dict with task_index(from depends_on) to task class
-    M   params:     same as extra[params] but with the resolved dependencies
-    M   vim_interfaces: similar to extra[interfaces] but with VIM information. Stored at database in the instance_XXX but not at vim_wim_actions
-    M   vim_info:   Detailed information of a vm,net from the VIM. Stored at database in the instance_XXX but not at vim_wim_actions
+            vim_info:   Detailed information of a vm/net from the VIM. Stored at database in the instance_XXX but not at
+                        vim_wim_actions
+    M   depends:    dict with task_index(from depends_on) to vim_id
+    M   params:     same as extra[params]
     MD  error_msg:  descriptive text upon an error.Stored also at database instance_XXX
-    MD  created_at: task creation time
-    MD  modified_at: last task update time. On refresh it contains when this task need to be refreshed
+    MD  created_at: task creation time. The task of creation must be the oldest
+    MD  modified_at: next time task need to be processed. For example, for a refresh, it contain next time refresh must
+                     be done
+    MD related:     All the tasks over the same VIM element have same "related". Note that other VIMs can contain the
+                    same value of related, but this thread only process those task of one VIM.  Also related can be the
+                    same among several NS os isntance-scenarios
+    MD worker:      Used to lock in case of several thread workers.
 
 """
 
@@ -99,6 +118,8 @@ class VimThreadExceptionNotFound(VimThreadException):
 class vim_thread(threading.Thread):
     REFRESH_BUILD = 5  # 5 seconds
     REFRESH_ACTIVE = 60  # 1 minute
+    REFRESH_ERROR = 600
+    REFRESH_DELETE = 3600 * 10
 
     def __init__(self, task_lock, name=None, datacenter_name=None, datacenter_tenant_id=None,
                  db=None, db_lock=None, ovim=None):
@@ -120,6 +141,7 @@ class vim_thread(threading.Thread):
         else:
             self.name = name
         self.vim_persistent_info = {}
+        self.my_id = self.name[:64]
 
         self.logger = logging.getLogger('openmano.vim.' + self.name)
         self.db = db
@@ -128,19 +150,6 @@ class vim_thread(threading.Thread):
         self.task_lock = task_lock
         self.task_queue = Queue.Queue(2000)
 
-        self.refresh_tasks = []
-        """Contains time ordered task list for refreshing the status of VIM VMs and nets"""
-
-        self.pending_tasks = []
-        """Contains time ordered task list for creation, deletion of VIM VMs and nets"""
-
-        self.grouped_tasks = {}
-        """ It contains all the creation/deletion pending tasks grouped by its concrete vm, net, etc
-            <item><item_id>:
-                -   <task1>  # e.g. CREATE task
-                    <task2>  # e.g. DELETE task
-        """
-
     def get_vimconnector(self):
         try:
             from_ = "datacenter_tenants as dt join datacenters as d on dt.datacenter_id=d.uuid"
@@ -177,363 +186,343 @@ class vim_thread(threading.Thread):
             self.vim = None
             self.error_status = "Error loading vimconnector: {}".format(e)
 
-    def _reload_vim_actions(self):
+    def _get_db_task(self):
         """
         Read actions from database and reload them at memory. Fill self.refresh_list, pending_list, vim_actions
         :return: None
         """
+        now = time.time()
         try:
-            action_completed = False
-            task_list = []
-            old_action_key = None
-
-            old_item_id = ""
-            old_item = ""
-            old_created_at = 0.0
-            database_limit = 200
+            database_limit = 20
+            task_related = None
             while True:
-                # get 200 (database_limit) entries each time
+                # get 20 (database_limit) entries each time
                 vim_actions = self.db.get_rows(FROM="vim_wim_actions",
-                                                WHERE={"datacenter_vim_id": self.datacenter_tenant_id,
-                                                        "item_id>=": old_item_id},
-                                                ORDER_BY=("item_id", "item", "created_at",),
-                                                LIMIT=database_limit)
+                                               WHERE={"datacenter_vim_id": self.datacenter_tenant_id,
+                                                      "status": ['SCHEDULED', 'BUILD', 'DONE'],
+                                                      "worker": [None, self.my_id], "modified_at<=": now
+                                                      },
+                                               ORDER_BY=("modified_at", "created_at",),
+                                               LIMIT=database_limit)
+                if not vim_actions:
+                    return None, None
+                # if vim_actions[0]["modified_at"] > now:
+                #     return int(vim_actions[0] - now)
                 for task in vim_actions:
-                    item = task["item"]
-                    item_id = task["item_id"]
-
-                    # skip the first entries that are already processed in the previous pool of 200
-                    if old_item_id:
-                        if item_id == old_item_id and item == old_item and task["created_at"] == old_created_at:
-                            old_item_id = False  # next one will be a new un-processed task
+                    # block related task
+                    if task_related == task["related"]:
+                        continue  # ignore if a locking has already tried for these task set
+                    task_related = task["related"]
+                    # lock ...
+                    self.db.update_rows("vim_wim_actions", UPDATE={"worker": self.my_id}, modified_time=0,
+                                        WHERE={"datacenter_vim_id": self.datacenter_tenant_id,
+                                               "status": ['SCHEDULED', 'BUILD', 'DONE', 'FAILED'],
+                                               "worker": [None, self.my_id],
+                                               "related": task_related,
+                                               "item": task["item"],
+                                               })
+                    # ... and read all related and check if locked
+                    related_tasks = self.db.get_rows(FROM="vim_wim_actions",
+                                                     WHERE={"datacenter_vim_id": self.datacenter_tenant_id,
+                                                            "status": ['SCHEDULED', 'BUILD', 'DONE', 'FAILED'],
+                                                            "related": task_related,
+                                                            "item": task["item"],
+                                                            },
+                                                     ORDER_BY=("created_at",))
+                    # check that all related tasks have been locked. If not release and try again. It can happen
+                    # for race conditions if a new related task has been inserted by nfvo in the process
+                    some_tasks_locked = False
+                    some_tasks_not_locked = False
+                    creation_task = None
+                    for relate_task in related_tasks:
+                        if relate_task["worker"] != self.my_id:
+                            some_tasks_not_locked = True
+                        else:
+                            some_tasks_locked = True
+                        if not creation_task and relate_task["action"] in ("CREATE", "FIND"):
+                            creation_task = relate_task
+                    if some_tasks_not_locked:
+                        if some_tasks_locked:  # unlock
+                            self.db.update_rows("vim_wim_actions", UPDATE={"worker": None}, modified_time=0,
+                                                WHERE={"datacenter_vim_id": self.datacenter_tenant_id,
+                                                       "worker": self.my_id,
+                                                       "related": task_related,
+                                                       "item": task["item"],
+                                                       })
                         continue
 
-                    action_key = item + item_id
-                    if old_action_key != action_key:
-                        if not action_completed and task_list:
-                            # This will fill needed task parameters into memory, and insert the task if needed in
-                            # self.pending_tasks or self.refresh_tasks
-                            try:
-                                self._insert_pending_tasks(task_list)
-                            except Exception as e:
-                                self.logger.critical(
-                                    "Unexpected exception at _reload_vim_actions:_insert_pending_tasks: " + str(e),
-                                    exc_info=True)
-                        task_list = []
-                        old_action_key = action_key
-                        action_completed = False
-                    elif action_completed:
-                        continue
+                    # task of creation must be the first in the list of related_task
+                    assert(related_tasks[0]["action"] in ("CREATE", "FIND"))
 
-                    if task["status"] == "SCHEDULED" or task["action"] == "CREATE" or task["action"] == "FIND":
-                        task_list.append(task)
-                    elif task["action"] == "DELETE":
-                        # action completed because deleted and status is not SCHEDULED. Not needed anything
-                        action_completed = True
-                if len(vim_actions) == database_limit:
-                    # update variables for get the next database iteration
-                    old_item_id = item_id
-                    old_item = item
-                    old_created_at = task["created_at"]
-                else:
-                    break
-            # Last actions group need to be inserted too
-            if not action_completed and task_list:
-                try:
-                    self._insert_pending_tasks(task_list)
-                except Exception as e:
-                    self.logger.critical("Unexpected exception at _reload_vim_actions:_insert_pending_tasks: " + str(e),
-                                         exc_info=True)
-            self.logger.debug("reloaded vim actions pending:{} refresh:{}".format(
-                len(self.pending_tasks), len(self.refresh_tasks)))
+                    if task["extra"]:
+                        extra = yaml.load(task["extra"])
+                    else:
+                        extra = {}
+                    task["extra"] = extra
+                    if extra.get("depends_on"):
+                        task["depends"] = {}
+                    if extra.get("params"):
+                        task["params"] = deepcopy(extra["params"])
+                    return task, related_tasks
         except Exception as e:
-            self.logger.critical("Unexpected exception at _reload_vim_actions: " + str(e), exc_info=True)
+            self.logger.critical("Unexpected exception at _get_db_task: " + str(e), exc_info=True)
+            return None, None
 
-    def _refres_elements(self):
-        """Call VIM to get VMs and networks status until 10 elements"""
-        now = time.time()
-        nb_processed = 0
-        vm_to_refresh_list = []
-        net_to_refresh_list = []
-        vm_to_refresh_dict = {}
-        net_to_refresh_dict = {}
-        items_to_refresh = 0
-        while self.refresh_tasks:
-            task = self.refresh_tasks[0]
-            with self.task_lock:
-                if task['status'] == 'SUPERSEDED':
-                    self.refresh_tasks.pop(0)
-                    continue
-                if task['modified_at'] > now:
+    def _delete_task(self, task):
+        """
+        Determine if this task need to be done or superseded
+        :return: None
+        """
+
+        def copy_extra_created(copy_to, copy_from):
+            copy_to["created"] = copy_from["created"]
+            if copy_from.get("sdn_net_id"):
+                copy_to["sdn_net_id"] = copy_from["sdn_net_id"]
+            if copy_from.get("interfaces"):
+                copy_to["interfaces"] = copy_from["interfaces"]
+            if copy_from.get("created_items"):
+                if not copy_to.get("created_items"):
+                    copy_to["created_items"] = {}
+                copy_to["created_items"].update(copy_from["created_items"])
+
+        task_create = None
+        dependency_task = None
+        deletion_needed = False
+        if task["status"] == "FAILED":
+            return   # TODO need to be retry??
+        try:
+            # get all related tasks
+            related_tasks = self.db.get_rows(FROM="vim_wim_actions",
+                                             WHERE={"datacenter_vim_id": self.datacenter_tenant_id,
+                                                    "status": ['SCHEDULED', 'BUILD', 'DONE', 'FAILED'],
+                                                    "action": ["FIND", "CREATE"],
+                                                    "related": task["related"],
+                                                    },
+                                             ORDER_BY=("created_at",),
+                                             )
+            for related_task in related_tasks:
+                if related_task["item"] == task["item"] and related_task["item_id"] == task["item_id"]:
+                    task_create = related_task
+                    # TASK_CREATE
+                    if related_task["extra"]:
+                        extra_created = yaml.load(related_task["extra"])
+                        if extra_created.get("created"):
+                            deletion_needed = True
+                        related_task["extra"] = extra_created
+                elif not dependency_task:
+                    dependency_task = related_task
+                if task_create and dependency_task:
                     break
-                # task["status"] = "processing"
-                nb_processed += 1
-            self.refresh_tasks.pop(0)
-            if task["item"] == 'instance_vms':
-                if task["vim_id"] not in vm_to_refresh_dict:
-                    vm_to_refresh_dict[task["vim_id"]] = [task]
-                    vm_to_refresh_list.append(task["vim_id"])
-                else:
-                    vm_to_refresh_dict[task["vim_id"]].append(task)
-            elif task["item"] == 'instance_nets':
-                if task["vim_id"] not in net_to_refresh_dict:
-                    net_to_refresh_dict[task["vim_id"]] = [task]
-                    net_to_refresh_list.append(task["vim_id"])
-                else:
-                    net_to_refresh_dict[task["vim_id"]].append(task)
+
+            # mark task_create as FINISHED
+            self.db.update_rows("vim_wim_actions", UPDATE={"status": "FINISHED"},
+                                WHERE={"datacenter_vim_id": self.datacenter_tenant_id,
+                                       "instance_action_id": task_create["instance_action_id"],
+                                       "task_index": task_create["task_index"]
+                                       })
+            if not deletion_needed:
+                return
+            elif dependency_task:
+                # move create information  from task_create to relate_task
+                extra_new_created = yaml.load(dependency_task["extra"]) or {}
+                extra_new_created["created"] = extra_created["created"]
+                copy_extra_created(copy_to=extra_new_created, copy_from=extra_created)
+
+                self.db.update_rows("vim_wim_actions",
+                                    UPDATE={"extra": yaml.safe_dump(extra_new_created, default_flow_style=True,
+                                                                    width=256),
+                                            "vim_id": task_create.get("vim_id")},
+                                    WHERE={"datacenter_vim_id": self.datacenter_tenant_id,
+                                           "instance_action_id": dependency_task["instance_action_id"],
+                                           "task_index": dependency_task["task_index"]
+                                           })
+                return False
             else:
-                task_id = task["instance_action_id"] + "." + str(task["task_index"])
-                self.logger.critical("task={}: unknown task {}".format(task_id, task["item"]), exc_info=True)
-            items_to_refresh += 1
-            if items_to_refresh == 10:
-                break
-
-        if vm_to_refresh_list:
-            now = time.time()
-            try:
-                vim_dict = self.vim.refresh_vms_status(vm_to_refresh_list)
-            except vimconn.vimconnException as e:
-                # Mark all tasks at VIM_ERROR status
-                self.logger.error("task=several get-VM: vimconnException when trying to refresh vms " + str(e))
-                vim_dict = {}
-                for vim_id in vm_to_refresh_list:
-                    vim_dict[vim_id] = {"status": "VIM_ERROR", "error_msg": str(e)}
-
-            for vim_id, vim_info in vim_dict.items():
-
-                # look for task
-                for task in vm_to_refresh_dict[vim_id]:
-                    task_need_update = False
-                    task_id = task["instance_action_id"] + "." + str(task["task_index"])
-                    self.logger.debug("task={} get-VM: vim_vm_id={} result={}".format(task_id, task["vim_id"], vim_info))
-
-                    # check and update interfaces
-                    task_warning_msg = ""
-                    for interface in vim_info.get("interfaces", ()):
-                        vim_interface_id = interface["vim_interface_id"]
-                        if vim_interface_id not in task["extra"]["interfaces"]:
-                            self.logger.critical("task={} get-VM: Interface not found {} on task info {}".format(
-                                task_id, vim_interface_id, task["extra"]["interfaces"]), exc_info=True)
-                            continue
-                        task_interface = task["extra"]["interfaces"][vim_interface_id]
-                        task_vim_interface = task["vim_interfaces"].get(vim_interface_id)
-                        if task_vim_interface != interface:
-                            # delete old port
-                            if task_interface.get("sdn_port_id"):
-                                try:
-                                    with self.db_lock:
-                                        self.ovim.delete_port(task_interface["sdn_port_id"], idempotent=True)
-                                        task_interface["sdn_port_id"] = None
-                                        task_need_update = True
-                                except ovimException as e:
-                                    error_text = "ovimException deleting external_port={}: {}".format(
-                                        task_interface["sdn_port_id"], e)
-                                    self.logger.error("task={} get-VM: {}".format(task_id, error_text), exc_info=True)
-                                    task_warning_msg += error_text
-                                    # TODO Set error_msg at instance_nets instead of instance VMs
-
-                            # Create SDN port
-                            sdn_net_id = task_interface.get("sdn_net_id")
-                            if sdn_net_id and interface.get("compute_node") and interface.get("pci"):
-                                sdn_port_name = sdn_net_id + "." + task["vim_id"]
-                                sdn_port_name = sdn_port_name[:63]
-                                try:
-                                    with self.db_lock:
-                                        sdn_port_id = self.ovim.new_external_port(
-                                            {"compute_node": interface["compute_node"],
-                                                "pci": interface["pci"],
-                                                "vlan": interface.get("vlan"),
-                                                "net_id": sdn_net_id,
-                                                "region": self.vim["config"]["datacenter_id"],
-                                                "name": sdn_port_name,
-                                                "mac": interface.get("mac_address")})
-                                        task_interface["sdn_port_id"] = sdn_port_id
-                                        task_need_update = True
-                                except (ovimException, Exception) as e:
-                                    error_text = "ovimException creating new_external_port compute_node={}" \
-                                                 " pci={} vlan={} {}".format(
-                                        interface["compute_node"],
-                                        interface["pci"],
-                                        interface.get("vlan"), e)
-                                    self.logger.error("task={} get-VM: {}".format(task_id, error_text), exc_info=True)
-                                    task_warning_msg += error_text
-                                    # TODO Set error_msg at instance_nets instead of instance VMs
-
-                            self.db.update_rows(
-                                'instance_interfaces',
-                                UPDATE={"mac_address": interface.get("mac_address"),
-                                        "ip_address": interface.get("ip_address"),
-                                        "vim_interface_id": interface.get("vim_interface_id"),
-                                        "vim_info": interface.get("vim_info"),
-                                        "sdn_port_id": task_interface.get("sdn_port_id"),
-                                        "compute_node": interface.get("compute_node"),
-                                        "pci": interface.get("pci"),
-                                        "vlan": interface.get("vlan")},
-                                WHERE={'uuid': task_interface["iface_id"]})
-                            task["vim_interfaces"][vim_interface_id] = interface
-
-                    # check and update task and instance_vms database
-                    vim_info_error_msg = None
-                    if vim_info.get("error_msg"):
-                        vim_info_error_msg = self._format_vim_error_msg(vim_info["error_msg"] + task_warning_msg)
-                    elif task_warning_msg:
-                        vim_info_error_msg = self._format_vim_error_msg(task_warning_msg)
-                    task_vim_info = task.get("vim_info")
-                    task_error_msg = task.get("error_msg")
-                    task_vim_status = task["extra"].get("vim_status")
-                    if task_vim_status != vim_info["status"] or task_error_msg != vim_info_error_msg or \
-                            (vim_info.get("vim_info") and task_vim_info != vim_info["vim_info"]):
-                        temp_dict = {"status": vim_info["status"], "error_msg": vim_info_error_msg}
-                        if vim_info.get("vim_info"):
-                            temp_dict["vim_info"] = vim_info["vim_info"]
-                        self.db.update_rows('instance_vms', UPDATE=temp_dict, WHERE={"uuid": task["item_id"]})
-                        task["extra"]["vim_status"] = vim_info["status"]
-                        task["error_msg"] = vim_info_error_msg
-                        if vim_info.get("vim_info"):
-                            task["vim_info"] = vim_info["vim_info"]
-                        task_need_update = True
-
-                    if task_need_update:
-                        self.db.update_rows(
-                            'vim_wim_actions',
-                            UPDATE={"extra": yaml.safe_dump(task["extra"], default_flow_style=True, width=256),
-                                    "error_msg": task.get("error_msg"), "modified_at": now},
-                            WHERE={'instance_action_id': task['instance_action_id'],
-                                    'task_index': task['task_index']})
-                    if task["extra"].get("vim_status") == "BUILD":
-                        self._insert_refresh(task, now + self.REFRESH_BUILD)
-                    else:
-                        self._insert_refresh(task, now + self.REFRESH_ACTIVE)
+                task["vim_id"] = task_create["vim_id"]
+                copy_extra_created(copy_to=task["extra"], copy_from=task_create["extra"])
+                return True
 
-        if net_to_refresh_list:
-            now = time.time()
-            try:
-                vim_dict = self.vim.refresh_nets_status(net_to_refresh_list)
-            except vimconn.vimconnException as e:
-                # Mark all tasks at VIM_ERROR status
-                self.logger.error("task=several get-net: vimconnException when trying to refresh nets " + str(e))
-                vim_dict = {}
-                for vim_id in net_to_refresh_list:
-                    vim_dict[vim_id] = {"status": "VIM_ERROR", "error_msg": str(e)}
-
-            for vim_id, vim_info in vim_dict.items():
-                # look for task
-                for task in net_to_refresh_dict[vim_id]:
-                    task_id = task["instance_action_id"] + "." + str(task["task_index"])
-                    self.logger.debug("task={} get-net: vim_net_id={} result={}".format(task_id, task["vim_id"], vim_info))
-
-                    task_vim_info = task.get("vim_info")
-                    task_vim_status = task["extra"].get("vim_status")
-                    task_error_msg = task.get("error_msg")
-                    task_sdn_net_id = task["extra"].get("sdn_net_id")
-
-                    vim_info_status = vim_info["status"]
-                    vim_info_error_msg = vim_info.get("error_msg")
-                    # get ovim status
-                    if task_sdn_net_id:
-                        try:
-                            with self.db_lock:
-                                sdn_net = self.ovim.show_network(task_sdn_net_id)
-                        except (ovimException, Exception) as e:
-                            text_error = "ovimException getting network snd_net_id={}: {}".format(task_sdn_net_id, e)
-                            self.logger.error("task={} get-net: {}".format(task_id, text_error), exc_info=True)
-                            sdn_net = {"status": "ERROR", "last_error": text_error}
-                        if sdn_net["status"] == "ERROR":
-                            if not vim_info_error_msg:
-                                vim_info_error_msg = str(sdn_net.get("last_error"))
-                            else:
-                                vim_info_error_msg = "VIM_ERROR: {} && SDN_ERROR: {}".format(
-                                    self._format_vim_error_msg(vim_info_error_msg, 1024 // 2 - 14),
-                                    self._format_vim_error_msg(sdn_net["last_error"], 1024 // 2 - 14))
-                            vim_info_status = "ERROR"
-                        elif sdn_net["status"] == "BUILD":
-                            if vim_info_status == "ACTIVE":
-                                vim_info_status = "BUILD"
-
-                    # update database
-                    if vim_info_error_msg:
-                        vim_info_error_msg = self._format_vim_error_msg(vim_info_error_msg)
-                    if task_vim_status != vim_info_status or task_error_msg != vim_info_error_msg or \
-                            (vim_info.get("vim_info") and task_vim_info != vim_info["vim_info"]):
-                        task["extra"]["vim_status"] = vim_info_status
-                        task["error_msg"] = vim_info_error_msg
-                        if vim_info.get("vim_info"):
-                            task["vim_info"] = vim_info["vim_info"]
-                        temp_dict = {"status": vim_info_status, "error_msg": vim_info_error_msg}
-                        if vim_info.get("vim_info"):
-                            temp_dict["vim_info"] = vim_info["vim_info"]
-                        self.db.update_rows('instance_nets', UPDATE=temp_dict, WHERE={"uuid": task["item_id"]})
-                        self.db.update_rows(
-                            'vim_wim_actions',
-                            UPDATE={"extra": yaml.safe_dump(task["extra"], default_flow_style=True, width=256),
-                                    "error_msg": task.get("error_msg"), "modified_at": now},
-                            WHERE={'instance_action_id': task['instance_action_id'],
-                                    'task_index': task['task_index']})
-                    if task["extra"].get("vim_status") == "BUILD":
-                        self._insert_refresh(task, now + self.REFRESH_BUILD)
-                    else:
-                        self._insert_refresh(task, now + self.REFRESH_ACTIVE)
+        except Exception as e:
+            self.logger.critical("Unexpected exception at _delete_task: " + str(e), exc_info=True)
 
-        return nb_processed
+    def _refres_vm(self, task):
+        """Call VIM to get VMs status"""
+        database_update = None
+
+        vim_id = task["vim_id"]
+        vm_to_refresh_list = [vim_id]
+        try:
+            vim_dict = self.vim.refresh_vms_status(vm_to_refresh_list)
+            vim_info = vim_dict[vim_id]
+        except vimconn.vimconnException as e:
+            # Mark all tasks at VIM_ERROR status
+            self.logger.error("task=several get-VM: vimconnException when trying to refresh vms " + str(e))
+            vim_info = {"status": "VIM_ERROR", "error_msg": str(e)}
 
-    def _insert_refresh(self, task, threshold_time=None):
-        """Insert a task at list of refreshing elements. The refreshing list is ordered by threshold_time (task['modified_at']
-        It is assumed that this is called inside this thread
-        """
-        if not self.vim:
-            return
-        if not threshold_time:
-            threshold_time = time.time()
-        task["modified_at"] = threshold_time
-        task_name = task["item"][9:] + "-" + task["action"]
         task_id = task["instance_action_id"] + "." + str(task["task_index"])
-        for index in range(0, len(self.refresh_tasks)):
-            if self.refresh_tasks[index]["modified_at"] > threshold_time:
-                self.refresh_tasks.insert(index, task)
-                break
-        else:
-            index = len(self.refresh_tasks)
-            self.refresh_tasks.append(task)
-        self.logger.debug("task={} new refresh name={}, modified_at={} index={}".format(
-            task_id, task_name, task["modified_at"], index))
-
-    def _remove_refresh(self, task_name, vim_id):
-        """Remove a task with this name and vim_id from the list of refreshing elements.
-        It is assumed that this is called inside this thread outside _refres_elements method
-        Return True if self.refresh_list is modified, task is found
-        Return False if not found
-        """
-        index_to_delete = None
-        for index in range(0, len(self.refresh_tasks)):
-            if self.refresh_tasks[index]["name"] == task_name and self.refresh_tasks[index]["vim_id"] == vim_id:
-                index_to_delete = index
-                break
-        else:
-            return False
-        if not index_to_delete:
-            del self.refresh_tasks[index_to_delete]
-        return True
-
-    def _proccess_pending_tasks(self):
-        nb_created = 0
-        nb_processed = 0
-        while self.pending_tasks:
-            task = self.pending_tasks.pop(0)
-            nb_processed += 1
+        self.logger.debug("task={} get-VM: vim_vm_id={} result={}".format(task_id, task["vim_id"], vim_info))
+
+        # check and update interfaces
+        task_warning_msg = ""
+        for interface in vim_info.get("interfaces", ()):
+            vim_interface_id = interface["vim_interface_id"]
+            if vim_interface_id not in task["extra"]["interfaces"]:
+                self.logger.critical("task={} get-VM: Interface not found {} on task info {}".format(
+                    task_id, vim_interface_id, task["extra"]["interfaces"]), exc_info=True)
+                continue
+            task_interface = task["extra"]["interfaces"][vim_interface_id]
+            task_vim_interface = task_interface.get("vim_info")
+            if task_vim_interface != interface:
+                # delete old port
+                if task_interface.get("sdn_port_id"):
+                    try:
+                        with self.db_lock:
+                            self.ovim.delete_port(task_interface["sdn_port_id"], idempotent=True)
+                            task_interface["sdn_port_id"] = None
+                    except ovimException as e:
+                        error_text = "ovimException deleting external_port={}: {}".format(
+                            task_interface["sdn_port_id"], e)
+                        self.logger.error("task={} get-VM: {}".format(task_id, error_text), exc_info=True)
+                        task_warning_msg += error_text
+                        # TODO Set error_msg at instance_nets instead of instance VMs
+
+                # Create SDN port
+                sdn_net_id = task_interface.get("sdn_net_id")
+                if sdn_net_id and interface.get("compute_node") and interface.get("pci"):
+                    sdn_port_name = sdn_net_id + "." + task["vim_id"]
+                    sdn_port_name = sdn_port_name[:63]
+                    try:
+                        with self.db_lock:
+                            sdn_port_id = self.ovim.new_external_port(
+                                {"compute_node": interface["compute_node"],
+                                    "pci": interface["pci"],
+                                    "vlan": interface.get("vlan"),
+                                    "net_id": sdn_net_id,
+                                    "region": self.vim["config"]["datacenter_id"],
+                                    "name": sdn_port_name,
+                                    "mac": interface.get("mac_address")})
+                            task_interface["sdn_port_id"] = sdn_port_id
+                    except (ovimException, Exception) as e:
+                        error_text = "ovimException creating new_external_port compute_node={} pci={} vlan={} {}".\
+                            format(interface["compute_node"], interface["pci"], interface.get("vlan"), e)
+                        self.logger.error("task={} get-VM: {}".format(task_id, error_text), exc_info=True)
+                        task_warning_msg += error_text
+                        # TODO Set error_msg at instance_nets instead of instance VMs
+
+                self.db.update_rows('instance_interfaces',
+                                    UPDATE={"mac_address": interface.get("mac_address"),
+                                            "ip_address": interface.get("ip_address"),
+                                            "vim_interface_id": interface.get("vim_interface_id"),
+                                            "vim_info": interface.get("vim_info"),
+                                            "sdn_port_id": task_interface.get("sdn_port_id"),
+                                            "compute_node": interface.get("compute_node"),
+                                            "pci": interface.get("pci"),
+                                            "vlan": interface.get("vlan")},
+                                    WHERE={'uuid': task_interface["iface_id"]})
+                task_interface["vim_info"] = interface
+
+        # check and update task and instance_vms database
+        vim_info_error_msg = None
+        if vim_info.get("error_msg"):
+            vim_info_error_msg = self._format_vim_error_msg(vim_info["error_msg"] + task_warning_msg)
+        elif task_warning_msg:
+            vim_info_error_msg = self._format_vim_error_msg(task_warning_msg)
+        task_vim_info = task["extra"].get("vim_info")
+        task_error_msg = task.get("error_msg")
+        task_vim_status = task["extra"].get("vim_status")
+        if task_vim_status != vim_info["status"] or task_error_msg != vim_info_error_msg or \
+                (vim_info.get("vim_info") and task_vim_info != vim_info["vim_info"]):
+            database_update = {"status": vim_info["status"], "error_msg": vim_info_error_msg}
+            if vim_info.get("vim_info"):
+                database_update["vim_info"] = vim_info["vim_info"]
+
+            task["extra"]["vim_status"] = vim_info["status"]
+            task["error_msg"] = vim_info_error_msg
+            if vim_info.get("vim_info"):
+                task["extra"]["vim_info"] = vim_info["vim_info"]
+
+        return database_update
+
+    def _refres_net(self, task):
+        """Call VIM to get network status"""
+        database_update = None
+
+        vim_id = task["vim_id"]
+        net_to_refresh_list = [vim_id]
+        try:
+            vim_dict = self.vim.refresh_nets_status(net_to_refresh_list)
+            vim_info = vim_dict[vim_id]
+        except vimconn.vimconnException as e:
+            # Mark all tasks at VIM_ERROR status
+            self.logger.error("task=several get-net: vimconnException when trying to refresh nets " + str(e))
+            vim_info = {"status": "VIM_ERROR", "error_msg": str(e)}
+
+        task_id = task["instance_action_id"] + "." + str(task["task_index"])
+        self.logger.debug("task={} get-net: vim_net_id={} result={}".format(task_id, task["vim_id"], vim_info))
+
+        task_vim_info = task["extra"].get("vim_info")
+        task_vim_status = task["extra"].get("vim_status")
+        task_error_msg = task.get("error_msg")
+        task_sdn_net_id = task["extra"].get("sdn_net_id")
+
+        vim_info_status = vim_info["status"]
+        vim_info_error_msg = vim_info.get("error_msg")
+        # get ovim status
+        if task_sdn_net_id:
             try:
+                with self.db_lock:
+                    sdn_net = self.ovim.show_network(task_sdn_net_id)
+            except (ovimException, Exception) as e:
+                text_error = "ovimException getting network snd_net_id={}: {}".format(task_sdn_net_id, e)
+                self.logger.error("task={} get-net: {}".format(task_id, text_error), exc_info=True)
+                sdn_net = {"status": "ERROR", "last_error": text_error}
+            if sdn_net["status"] == "ERROR":
+                if not vim_info_error_msg:
+                    vim_info_error_msg = str(sdn_net.get("last_error"))
+                else:
+                    vim_info_error_msg = "VIM_ERROR: {} && SDN_ERROR: {}".format(
+                        self._format_vim_error_msg(vim_info_error_msg, 1024 // 2 - 14),
+                        self._format_vim_error_msg(sdn_net["last_error"], 1024 // 2 - 14))
+                vim_info_status = "ERROR"
+            elif sdn_net["status"] == "BUILD":
+                if vim_info_status == "ACTIVE":
+                    vim_info_status = "BUILD"
+
+        # update database
+        if vim_info_error_msg:
+            vim_info_error_msg = self._format_vim_error_msg(vim_info_error_msg)
+        if task_vim_status != vim_info_status or task_error_msg != vim_info_error_msg or \
+                (vim_info.get("vim_info") and task_vim_info != vim_info["vim_info"]):
+            task["extra"]["vim_status"] = vim_info_status
+            task["error_msg"] = vim_info_error_msg
+            if vim_info.get("vim_info"):
+                task["extra"]["vim_info"] = vim_info["vim_info"]
+            database_update = {"status": vim_info_status, "error_msg": vim_info_error_msg}
+            if vim_info.get("vim_info"):
+                database_update["vim_info"] = vim_info["vim_info"]
+        return database_update
+
+    def _proccess_pending_tasks(self, task, related_tasks):
+        old_task_status = task["status"]
+        create_or_find = False   # if as result of processing this task something is created or found
+        next_refresh = 0
+
+        try:
+            if task["status"] == "SCHEDULED":
                 # check if tasks that this depends on have been completed
                 dependency_not_completed = False
+                dependency_modified_at = 0
                 for task_index in task["extra"].get("depends_on", ()):
-                    task_dependency = task["depends"].get("TASK-" + str(task_index))
+                    task_dependency = self._look_for_task(task["instance_action_id"], task_index)
                     if not task_dependency:
-                        task_dependency = self._look_for_task(task["instance_action_id"], task_index)
-                        if not task_dependency:
-                            raise VimThreadException(
-                                "Cannot get depending net task trying to get depending task {}.{}".format(
-                                    task["instance_action_id"], task_index))
-                        # task["depends"]["TASK-" + str(task_index)] = task_dependency #it references another object,so database must be look again
+                        raise VimThreadException(
+                            "Cannot get depending net task trying to get depending task {}.{}".format(
+                                task["instance_action_id"], task_index))
+                    # task["depends"]["TASK-" + str(task_index)] = task_dependency #it references another object,so
+                    # database must be look again
                     if task_dependency["status"] == "SCHEDULED":
                         dependency_not_completed = True
+                        dependency_modified_at = task_dependency["modified_at"]
                         break
                     elif task_dependency["status"] == "FAILED":
                         raise VimThreadException(
@@ -542,205 +531,184 @@ class vim_thread(threading.Thread):
                                 task["instance_action_id"], task["task_index"],
                                 task_dependency["instance_action_id"], task_dependency["task_index"],
                                 task_dependency["action"], task_dependency["item"], task_dependency.get("error_msg")))
+
+                    task["depends"]["TASK-"+str(task_index)] = task_dependency["vim_id"]
+                    task["depends"]["TASK-{}.{}".format(task["instance_action_id"], task_index)] =\
+                        task_dependency["vim_id"]
                 if dependency_not_completed:
-                    # Move this task to the end.
-                    task["extra"]["tries"] = task["extra"].get("tries", 0) + 1
-                    if task["extra"]["tries"] <= 3:
-                        self.pending_tasks.append(task)
-                        continue
-                    else:
-                        raise VimThreadException(
-                            "Cannot {} {}, (task {}.{}) because timeout waiting to complete {} {}, "
-                            "(task {}.{})".format(task["action"], task["item"],
-                                                  task["instance_action_id"], task["task_index"],
-                                                  task_dependency["instance_action_id"], task_dependency["task_index"],
-                                                  task_dependency["action"], task_dependency["item"]))
-
-                if task["status"] == "SUPERSEDED":
-                    # not needed to do anything but update database with the new status
-                    result = True
-                    database_update = None
-                elif not self.vim:
-                    task["status"] = "ERROR"
-                    task["error_msg"] = self.error_status
-                    result = False
-                    database_update = {"status": "VIM_ERROR", "error_msg": task["error_msg"]}
-                elif task["item"] == 'instance_vms':
-                    if task["action"] == "CREATE":
-                        result, database_update = self.new_vm(task)
-                        nb_created += 1
-                    elif task["action"] == "DELETE":
-                        result, database_update = self.del_vm(task)
-                    else:
-                        raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
-                elif task["item"] == 'instance_nets':
-                    if task["action"] == "CREATE":
-                        result, database_update = self.new_net(task)
-                        nb_created += 1
-                    elif task["action"] == "DELETE":
-                        result, database_update = self.del_net(task)
-                    elif task["action"] == "FIND":
-                        result, database_update = self.get_net(task)
-                    else:
-                        raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
-                elif task["item"] == 'instance_sfis':
-                    if task["action"] == "CREATE":
-                        result, database_update = self.new_sfi(task)
-                        nb_created += 1
-                    elif task["action"] == "DELETE":
-                        result, database_update = self.del_sfi(task)
-                    else:
-                        raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
-                elif task["item"] == 'instance_sfs':
-                    if task["action"] == "CREATE":
-                        result, database_update = self.new_sf(task)
-                        nb_created += 1
-                    elif task["action"] == "DELETE":
-                        result, database_update = self.del_sf(task)
-                    else:
-                        raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
-                elif task["item"] == 'instance_classifications':
-                    if task["action"] == "CREATE":
-                        result, database_update = self.new_classification(task)
-                        nb_created += 1
-                    elif task["action"] == "DELETE":
-                        result, database_update = self.del_classification(task)
-                    else:
-                        raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
-                elif task["item"] == 'instance_sfps':
-                    if task["action"] == "CREATE":
-                        result, database_update = self.new_sfp(task)
-                        nb_created += 1
-                    elif task["action"] == "DELETE":
-                        result, database_update = self.del_sfp(task)
-                    else:
-                        raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
-                else:
-                    raise vimconn.vimconnException(self.name + "unknown task item {}".format(task["item"]))
-                    # TODO
-            except VimThreadException as e:
-                result = False
-                task["error_msg"] = str(e)
+                    # Move this task to the time dependency is going to be modified plus 10 seconds.
+                    self.db.update_rows("vim_wim_actions", modified_time=dependency_modified_at + 10,
+                                        UPDATE={"worker": None},
+                                        WHERE={"datacenter_vim_id": self.datacenter_tenant_id, "worker": self.my_id,
+                                               "related": task["related"],
+                                               })
+                    # task["extra"]["tries"] = task["extra"].get("tries", 0) + 1
+                    # if task["extra"]["tries"] > 3:
+                    #     raise VimThreadException(
+                    #         "Cannot {} {}, (task {}.{}) because timeout waiting to complete {} {}, "
+                    #         "(task {}.{})".format(task["action"], task["item"],
+                    #                               task["instance_action_id"], task["task_index"],
+                    #                               task_dependency["instance_action_id"], task_dependency["task_index"]
+                    #                               task_dependency["action"], task_dependency["item"]))
+                    return
+
+            database_update = None
+            if task["action"] == "DELETE":
+                deleted_needed = self._delete_task(task)
+                if not deleted_needed:
+                    task["status"] = "SUPERSEDED"  # with FINISHED instead of DONE it will not be refreshing
+                    task["error_msg"] = None
+
+            if task["status"] == "SUPERSEDED":
+                # not needed to do anything but update database with the new status
+                database_update = None
+            elif not self.vim:
                 task["status"] = "FAILED"
+                task["error_msg"] = self.error_status
                 database_update = {"status": "VIM_ERROR", "error_msg": task["error_msg"]}
+            elif task["item_id"] != related_tasks[0]["item_id"] and task["action"] in ("FIND", "CREATE"):
+                # Do nothing, just copy values from one to another and updata database
+                task["status"] = related_tasks[0]["status"]
+                task["error_msg"] = related_tasks[0]["error_msg"]
+                task["vim_id"] = related_tasks[0]["vim_id"]
+                extra = yaml.load(related_tasks[0]["extra"])
+                task["extra"]["vim_status"] = extra["vim_status"]
+                next_refresh = related_tasks[0]["modified_at"] + 0.001
+                database_update = {"status": task["extra"].get("vim_status", "VIM_ERROR"),
+                                   "error_msg": task["error_msg"]}
                 if task["item"] == 'instance_vms':
-                    database_update["vim_vm_id"] = None
+                    database_update["vim_vm_id"] = task["vim_id"]
                 elif task["item"] == 'instance_nets':
-                    database_update["vim_net_id"] = None
+                    database_update["vim_net_id"] = task["vim_id"]
+            elif task["item"] == 'instance_vms':
+                if task["status"] in ('BUILD', 'DONE') and task["action"] in ("FIND", "CREATE"):
+                    database_update = self._refres_vm(task)
+                    create_or_find = True
+                elif task["action"] == "CREATE":
+                    create_or_find = True
+                    database_update = self.new_vm(task)
+                elif task["action"] == "DELETE":
+                    self.del_vm(task)
+                else:
+                    raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
+            elif task["item"] == 'instance_nets':
+                if task["status"] in ('BUILD', 'DONE') and task["action"] in ("FIND", "CREATE"):
+                    database_update = self._refres_net(task)
+                    create_or_find = True
+                elif task["action"] == "CREATE":
+                    create_or_find = True
+                    database_update = self.new_net(task)
+                elif task["action"] == "DELETE":
+                    self.del_net(task)
+                elif task["action"] == "FIND":
+                    database_update = self.get_net(task)
+                else:
+                    raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
+            elif task["item"] == 'instance_sfis':
+                if task["action"] == "CREATE":
+                    create_or_find = True
+                    database_update = self.new_sfi(task)
+                elif task["action"] == "DELETE":
+                    self.del_sfi(task)
+                else:
+                    raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
+            elif task["item"] == 'instance_sfs':
+                if task["action"] == "CREATE":
+                    create_or_find = True
+                    database_update = self.new_sf(task)
+                elif task["action"] == "DELETE":
+                    self.del_sf(task)
+                else:
+                    raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
+            elif task["item"] == 'instance_classifications':
+                if task["action"] == "CREATE":
+                    create_or_find = True
+                    database_update = self.new_classification(task)
+                elif task["action"] == "DELETE":
+                    self.del_classification(task)
+                else:
+                    raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
+            elif task["item"] == 'instance_sfps':
+                if task["action"] == "CREATE":
+                    create_or_find = True
+                    database_update = self.new_sfp(task)
+                elif task["action"] == "DELETE":
+                    self.del_sfp(task)
+                else:
+                    raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
+            else:
+                raise vimconn.vimconnException(self.name + "unknown task item {}".format(task["item"]))
+                # TODO
+        except VimThreadException as e:
+            task["error_msg"] = str(e)
+            task["status"] = "FAILED"
+            database_update = {"status": "VIM_ERROR", "error_msg": task["error_msg"]}
+            if task["item"] == 'instance_vms':
+                database_update["vim_vm_id"] = None
+            elif task["item"] == 'instance_nets':
+                database_update["vim_net_id"] = None
 
-            no_refresh_tasks = ['instance_sfis', 'instance_sfs',
-                                'instance_classifications', 'instance_sfps']
-            if task["action"] == "DELETE":
-                action_key = task["item"] + task["item_id"]
-                del self.grouped_tasks[action_key]
-            elif task["action"] in ("CREATE", "FIND") and task["status"] in ("DONE", "BUILD"):
-                if task["item"] not in no_refresh_tasks:
-                    self._insert_refresh(task)
+        task_id = task["instance_action_id"] + "." + str(task["task_index"])
+        self.logger.debug("task={} item={} action={} result={}:'{}' params={}".format(
+            task_id, task["item"], task["action"], task["status"],
+            task["vim_id"] if task["status"] == "DONE" else task.get("error_msg"), task["params"]))
+        try:
+            if not next_refresh:
+                if task["status"] == "DONE":
+                    next_refresh = time.time()
+                    if task["extra"].get("vim_status") == "BUILD":
+                        next_refresh += self.REFRESH_BUILD
+                    elif task["extra"].get("vim_status") in ("ERROR", "VIM_ERROR"):
+                        next_refresh += self.REFRESH_ERROR
+                    elif task["extra"].get("vim_status") == "DELETED":
+                        next_refresh += self.REFRESH_DELETE
+                    else:
+                        next_refresh += self.REFRESH_ACTIVE
+                elif task["status"] == "FAILED":
+                    next_refresh = time.time() + self.REFRESH_DELETE
 
-            task_id = task["instance_action_id"] + "." + str(task["task_index"])
-            self.logger.debug("task={} item={} action={} result={}:'{}' params={}".format(
-                task_id, task["item"], task["action"], task["status"],
-                task["vim_id"] if task["status"] == "DONE" else task.get("error_msg"), task["params"]))
-            try:
-                now = time.time()
+            if create_or_find:
+                # modify all related task with action FIND/CREATED non SCHEDULED
                 self.db.update_rows(
-                    table="vim_wim_actions",
-                    UPDATE={"status": task["status"], "vim_id": task.get("vim_id"), "modified_at": now,
+                    table="vim_wim_actions", modified_time=next_refresh + 0.001,
+                    UPDATE={"status": task["status"], "vim_id": task.get("vim_id"),
                             "error_msg": task["error_msg"],
-                            "extra": yaml.safe_dump(task["extra"], default_flow_style=True, width=256)},
-                    WHERE={"instance_action_id": task["instance_action_id"], "task_index": task["task_index"]})
-                if result is not None:
-                    self.db.update_rows(
-                        table="instance_actions",
-                        UPDATE={("number_done" if result else "number_failed"): {"INCREMENT": 1},
-                                "modified_at": now},
-                        WHERE={"uuid": task["instance_action_id"]})
-                if database_update:
-                    self.db.update_rows(table=task["item"],
-                                        UPDATE=database_update,
-                                        WHERE={"uuid": task["item_id"]})
-            except db_base_Exception as e:
-                self.logger.error("task={} Error updating database {}".format(task_id, e), exc_info=True)
-
-            if nb_created == 10:
-                break
-        return nb_processed
-
-    def _insert_pending_tasks(self, vim_actions_list):
-        for task in vim_actions_list:
-            if task["datacenter_vim_id"] != self.datacenter_tenant_id:
-                continue
-            item = task["item"]
-            item_id = task["item_id"]
-            action_key = item + item_id
-            if action_key not in self.grouped_tasks:
-                self.grouped_tasks[action_key] = []
-            task["params"] = None
-            task["depends"] = {}
-            if task["extra"]:
-                extra = yaml.load(task["extra"])
-                task["extra"] = extra
-                task["params"] = extra.get("params")
-                depends_on_list = extra.get("depends_on")
-                if depends_on_list:
-                    for dependency_task in depends_on_list:
-                        if isinstance(dependency_task, int):
-                            index = dependency_task
-                        else:
-                            instance_action_id, _, task_id = dependency_task.rpartition(".")
-                            if instance_action_id != task["instance_action_id"]:
-                                continue
-                            index = int(task_id)
-
-                        if index < len(vim_actions_list) and vim_actions_list[index]["task_index"] == index and \
-                                vim_actions_list[index]["instance_action_id"] == task["instance_action_id"]:
-                            task["depends"]["TASK-" + str(index)] = vim_actions_list[index]
-                            task["depends"]["TASK-{}.{}".format(task["instance_action_id"], index)] = vim_actions_list[index]
-                if extra.get("interfaces"):
-                    task["vim_interfaces"] = {}
-            else:
-                task["extra"] = {}
-            if "error_msg" not in task:
-                task["error_msg"] = None
-            if "vim_id" not in task:
-                task["vim_id"] = None
-
-            if task["action"] == "DELETE":
-                need_delete_action = False
-                for to_supersede in self.grouped_tasks.get(action_key, ()):
-                    if to_supersede["action"] == "FIND" and to_supersede.get("vim_id"):
-                        task["vim_id"] = to_supersede["vim_id"]
-                    if to_supersede["action"] == "CREATE" and to_supersede["extra"].get("created", True) and \
-                            (to_supersede.get("vim_id") or to_supersede["extra"].get("sdn_net_id")):
-                        need_delete_action = True
-                        task["vim_id"] = to_supersede["vim_id"]
-                        if to_supersede["extra"].get("sdn_net_id"):
-                            task["extra"]["sdn_net_id"] = to_supersede["extra"]["sdn_net_id"]
-                        if to_supersede["extra"].get("interfaces"):
-                            task["extra"]["interfaces"] = to_supersede["extra"]["interfaces"]
-                        if to_supersede["extra"].get("created_items"):
-                            if not task["extra"].get("created_items"):
-                                task["extra"]["created_items"] = {}
-                            task["extra"]["created_items"].update(to_supersede["extra"]["created_items"])
-                    # Mark task as SUPERSEDED.
-                    #   If task is in self.pending_tasks, it will be removed and database will be update
-                    #   If task is in self.refresh_tasks, it will be removed
-                    to_supersede["status"] = "SUPERSEDED"
-                if not need_delete_action:
-                    task["status"] = "SUPERSEDED"
-
-                self.grouped_tasks[action_key].append(task)
-                self.pending_tasks.append(task)
-            elif task["status"] == "SCHEDULED":
-                self.grouped_tasks[action_key].append(task)
-                self.pending_tasks.append(task)
-            elif task["action"] in ("CREATE", "FIND"):
-                self.grouped_tasks[action_key].append(task)
-                if task["status"] in ("DONE", "BUILD"):
-                    self._insert_refresh(task)
-            # TODO add VM reset, get console, etc...
-            else:
-                raise vimconn.vimconnException(self.name + "unknown vim_action action {}".format(task["action"]))
+                            },
+
+                    WHERE={"datacenter_vim_id": self.datacenter_tenant_id,
+                           "worker": self.my_id,
+                           "action": ["FIND", "CREATE"],
+                           "related": task["related"],
+                           "status<>": "SCHEDULED",
+                           })
+            # modify own task
+            self.db.update_rows(
+                table="vim_wim_actions", modified_time=next_refresh,
+                UPDATE={"status": task["status"], "vim_id": task.get("vim_id"),
+                        "error_msg": task["error_msg"],
+                        "extra": yaml.safe_dump(task["extra"], default_flow_style=True, width=256)},
+                WHERE={"instance_action_id": task["instance_action_id"], "task_index": task["task_index"]})
+            # Unlock tasks
+            self.db.update_rows(
+                table="vim_wim_actions", modified_time=0,
+                UPDATE={"worker": None},
+                WHERE={"datacenter_vim_id": self.datacenter_tenant_id,
+                       "worker": self.my_id,
+                       "related": task["related"],
+                       })
+
+            # Update table instance_actions
+            if old_task_status == "SCHEDULED" and task["status"] != old_task_status:
+                self.db.update_rows(
+                    table="instance_actions",
+                    UPDATE={("number_failed" if task["status"] == "FAILED" else "number_done"): {"INCREMENT": 1}},
+                    WHERE={"uuid": task["instance_action_id"]})
+            if database_update:
+                self.db.update_rows(table=task["item"],
+                                    UPDATE=database_update,
+                                    WHERE={"related": task["related"]})
+        except db_base_Exception as e:
+            self.logger.error("task={} Error updating database {}".format(task_id, e), exc_info=True)
 
     def insert_task(self, task):
         try:
@@ -763,7 +731,6 @@ class vim_thread(threading.Thread):
         while True:
             self.get_vimconnector()
             self.logger.debug("Vimconnector loaded")
-            self._reload_vim_actions()
             reload_thread = False
 
             while True:
@@ -771,7 +738,7 @@ class vim_thread(threading.Thread):
                     while not self.task_queue.empty():
                         task = self.task_queue.get()
                         if isinstance(task, list):
-                            self._insert_pending_tasks(task)
+                            pass
                         elif isinstance(task, str):
                             if task == 'exit':
                                 return 0
@@ -781,10 +748,12 @@ class vim_thread(threading.Thread):
                         self.task_queue.task_done()
                     if reload_thread:
                         break
-                    nb_processed = self._proccess_pending_tasks()
-                    nb_processed += self._refres_elements()
-                    if not nb_processed:
-                        time.sleep(1)
+
+                    task, related_tasks = self._get_db_task()
+                    if task:
+                        self._proccess_pending_tasks(task, related_tasks)
+                    else:
+                        time.sleep(5)
 
                 except Exception as e:
                     self.logger.critical("Unexpected exception at run: " + str(e), exc_info=True)
@@ -811,7 +780,7 @@ class vim_thread(threading.Thread):
                 instance_action_id = ins_action_id
 
         tasks = self.db.get_rows(FROM="vim_wim_actions", WHERE={"instance_action_id": instance_action_id,
-                                                            "task_index": task_index})
+                                                                "task_index": task_index})
         if not tasks:
             return None
         task = tasks[0]
@@ -821,8 +790,6 @@ class vim_thread(threading.Thread):
             extra = yaml.load(task["extra"])
             task["extra"] = extra
             task["params"] = extra.get("params")
-            if extra.get("interfaces"):
-                task["vim_interfaces"] = {}
         else:
             task["extra"] = {}
         return task
@@ -841,14 +808,7 @@ class vim_thread(threading.Thread):
             net_list = params[5]
             for net in net_list:
                 if "net_id" in net and is_task_id(net["net_id"]):  # change task_id into network_id
-                    task_dependency = task["depends"].get(net["net_id"])
-                    if not task_dependency:
-                        task_dependency = self._look_for_task(task["instance_action_id"], net["net_id"])
-                        if not task_dependency:
-                            raise VimThreadException(
-                                "Cannot get depending net task trying to get depending task {}.{}".format(
-                                    task["instance_action_id"], net["net_id"]))
-                    network_id = task_dependency.get("vim_id")
+                    network_id = task["depends"][net["net_id"]]
                     if not network_id:
                         raise VimThreadException(
                             "Cannot create VM because depends on a network not created or found: " +
@@ -870,18 +830,19 @@ class vim_thread(threading.Thread):
                     task_interfaces[iface["vim_id"]]["interface_id"] = result[0]['interface_id']
                 else:
                     self.logger.critical("task={} new-VM: instance_nets uuid={} not found at DB".format(task_id,
-                                                                                                        iface["uuid"]), exc_info=True)
+                                                                                                        iface["uuid"]),
+                                         exc_info=True)
 
             task["vim_info"] = {}
-            task["vim_interfaces"] = {}
             task["extra"]["interfaces"] = task_interfaces
             task["extra"]["created"] = True
             task["extra"]["created_items"] = created_items
+            task["extra"]["vim_status"] = "BUILD"
             task["error_msg"] = None
             task["status"] = "DONE"
             task["vim_id"] = vim_vm_id
             instance_element_update = {"status": "BUILD", "vim_vm_id": vim_vm_id, "error_msg": None}
-            return True, instance_element_update
+            return instance_element_update
 
         except (vimconn.vimconnException, VimThreadException) as e:
             self.logger.error("task={} new-VM: {}".format(task_id, e))
@@ -890,7 +851,7 @@ class vim_thread(threading.Thread):
             task["status"] = "FAILED"
             task["vim_id"] = None
             instance_element_update = {"status": "VIM_ERROR", "vim_vm_id": None, "error_msg": error_text}
-            return False, instance_element_update
+            return instance_element_update
 
     def del_vm(self, task):
         task_id = task["instance_action_id"] + "." + str(task["task_index"])
@@ -908,18 +869,18 @@ class vim_thread(threading.Thread):
                         # TODO Set error_msg at instance_nets
 
             self.vim.delete_vminstance(vm_vim_id, task["extra"].get("created_items"))
-            task["status"] = "DONE"
+            task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
             task["error_msg"] = None
-            return True, None
+            return None
 
         except vimconn.vimconnException as e:
             task["error_msg"] = self._format_vim_error_msg(str(e))
             if isinstance(e, vimconn.vimconnNotFoundException):
                 # If not found mark as Done and fill error_msg
-                task["status"] = "DONE"
-                return True, None
+                task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
+                return None
             task["status"] = "FAILED"
-            return False, None
+            return None
 
     def _get_net_internal(self, task, filter_param):
         """
@@ -939,15 +900,15 @@ class vim_thread(threading.Thread):
         # Discover if this network is managed by a sdn controller
         sdn_net_id = None
         result = self.db.get_rows(SELECT=('sdn_net_id',), FROM='instance_nets',
-                                    WHERE={'vim_net_id': vim_net_id,
-                                            'datacenter_tenant_id': self.datacenter_tenant_id},
-                                    ORDER="instance_scenario_id")
+                                  WHERE={'vim_net_id': vim_net_id, 'datacenter_tenant_id': self.datacenter_tenant_id},
+                                  ORDER="instance_scenario_id")
         if result:
             sdn_net_id = result[0]['sdn_net_id']
 
         task["status"] = "DONE"
         task["extra"]["vim_info"] = {}
         task["extra"]["created"] = False
+        task["extra"]["vim_status"] = "BUILD"
         task["extra"]["sdn_net_id"] = sdn_net_id
         task["error_msg"] = None
         task["vim_id"] = vim_net_id
@@ -961,7 +922,7 @@ class vim_thread(threading.Thread):
             params = task["params"]
             filter_param = params[0]
             instance_element_update = self._get_net_internal(task, filter_param)
-            return True, instance_element_update
+            return instance_element_update
 
         except (vimconn.vimconnException, VimThreadException) as e:
             self.logger.error("task={} get-net: {}".format(task_id, e))
@@ -970,7 +931,7 @@ class vim_thread(threading.Thread):
             task["error_msg"] = self._format_vim_error_msg(str(e))
             instance_element_update = {"vim_net_id": None, "status": "VIM_ERROR",
                                        "error_msg": task["error_msg"]}
-            return False, instance_element_update
+            return instance_element_update
 
     def new_net(self, task):
         vim_net_id = None
@@ -984,7 +945,7 @@ class vim_thread(threading.Thread):
                 filter_param = task["extra"]["find"][0]
                 try:
                     instance_element_update = self._get_net_internal(task, filter_param)
-                    return True, instance_element_update
+                    return instance_element_update
                 except VimThreadExceptionNotFound:
                     pass
             # CREATE
@@ -1034,17 +995,17 @@ class vim_thread(threading.Thread):
                             sdn_external_port_id = self.ovim.new_external_port(sdn_port_data)
                     self.logger.debug("Added sdn_external_port {} to sdn_network {}".format(sdn_external_port_id,
                                                                                             sdn_net_id))
-
             task["status"] = "DONE"
             task["extra"]["vim_info"] = {}
             task["extra"]["sdn_net_id"] = sdn_net_id
+            task["extra"]["vim_status"] = "BUILD"
             task["extra"]["created"] = True
             task["extra"]["created_items"] = created_items
             task["error_msg"] = None
             task["vim_id"] = vim_net_id
             instance_element_update = {"vim_net_id": vim_net_id, "sdn_net_id": sdn_net_id, "status": "BUILD",
                                        "created": True, "error_msg": None}
-            return True, instance_element_update
+            return instance_element_update
         except (vimconn.vimconnException, ovimException) as e:
             self.logger.error("task={} new-net: Error {}: {}".format(task_id, action_text, e))
             task["status"] = "FAILED"
@@ -1053,7 +1014,7 @@ class vim_thread(threading.Thread):
             task["extra"]["sdn_net_id"] = sdn_net_id
             instance_element_update = {"vim_net_id": vim_net_id, "sdn_net_id": sdn_net_id, "status": "VIM_ERROR",
                                        "error_msg": task["error_msg"]}
-            return False, instance_element_update
+            return instance_element_update
 
     def del_net(self, task):
         net_vim_id = task["vim_id"]
@@ -1070,9 +1031,9 @@ class vim_thread(threading.Thread):
                     self.ovim.delete_network(sdn_net_id, idempotent=True)
             if net_vim_id:
                 self.vim.delete_network(net_vim_id, task["extra"].get("created_items"))
-            task["status"] = "DONE"
+            task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
             task["error_msg"] = None
-            return True, None
+            return None
         except ovimException as e:
             task["error_msg"] = self._format_vim_error_msg("ovimException obtaining and deleting external "
                                                            "ports for net {}: {}".format(sdn_net_id, str(e)))
@@ -1080,13 +1041,12 @@ class vim_thread(threading.Thread):
             task["error_msg"] = self._format_vim_error_msg(str(e))
             if isinstance(e, vimconn.vimconnNotFoundException):
                 # If not found mark as Done and fill error_msg
-                task["status"] = "DONE"
-                return True, None
+                task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
+                return None
         task["status"] = "FAILED"
-        return False, None
-
-    ## Service Function Instances
+        return None
 
+    # Service Function Instances
     def new_sfi(self, task):
         vim_sfi_id = None
         try:
@@ -1112,12 +1072,16 @@ class vim_thread(threading.Thread):
             else:
                 egress_vim_interface_id = ingress_vim_interface_id
             if not ingress_vim_interface_id or not egress_vim_interface_id:
-                self.logger.error("Error creating Service Function Instance, Ingress: %s, Egress: %s",
-                                  ingress_vim_interface_id, egress_vim_interface_id)
-                return False, None
+                error_text = "Error creating Service Function Instance, Ingress: {}, Egress: {}".format(
+                    ingress_vim_interface_id, egress_vim_interface_id)
+                self.logger.error(error_text)
+                task["error_msg"] = error_text
+                task["status"] = "FAILED"
+                task["vim_id"] = None
+                return None
             # At the moment, every port associated with the VM will be used both as ingress and egress ports.
-            # Bear in mind that different VIM connectors might support SFI differently. In the case of OpenStack, only the
-            # first ingress and first egress ports will be used to create the SFI (Port Pair).
+            # Bear in mind that different VIM connectors might support SFI differently. In the case of OpenStack,
+            # only the first ingress and first egress ports will be used to create the SFI (Port Pair).
             ingress_port_id_list = [ingress_vim_interface_id]
             egress_port_id_list = [egress_vim_interface_id]
             name = "sfi-%s" % task["item_id"][:8]
@@ -1125,11 +1089,12 @@ class vim_thread(threading.Thread):
             vim_sfi_id = self.vim.new_sfi(name, ingress_port_id_list, egress_port_id_list, sfc_encap=False)
 
             task["extra"]["created"] = True
+            task["extra"]["vim_status"] = "ACTIVE"
             task["error_msg"] = None
-            task["status"] = "DONE"
+            task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
             task["vim_id"] = vim_sfi_id
             instance_element_update = {"status": "ACTIVE", "vim_sfi_id": vim_sfi_id, "error_msg": None}
-            return True, instance_element_update
+            return instance_element_update
 
         except (vimconn.vimconnException, VimThreadException) as e:
             self.logger.error("Error creating Service Function Instance, task=%s: %s", task_id, str(e))
@@ -1138,24 +1103,24 @@ class vim_thread(threading.Thread):
             task["status"] = "FAILED"
             task["vim_id"] = None
             instance_element_update = {"status": "VIM_ERROR", "vim_sfi_id": None, "error_msg": error_text}
-            return False, instance_element_update
+            return instance_element_update
 
     def del_sfi(self, task):
         sfi_vim_id = task["vim_id"]
         try:
             self.vim.delete_sfi(sfi_vim_id)
-            task["status"] = "DONE"
+            task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
             task["error_msg"] = None
-            return True, None
+            return None
 
         except vimconn.vimconnException as e:
             task["error_msg"] = self._format_vim_error_msg(str(e))
             if isinstance(e, vimconn.vimconnNotFoundException):
                 # If not found mark as Done and fill error_msg
-                task["status"] = "DONE"
-                return True, None
+                task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
+                return None
             task["status"] = "FAILED"
-            return False, None
+            return None
 
     def new_sf(self, task):
         vim_sf_id = None
@@ -1173,11 +1138,12 @@ class vim_thread(threading.Thread):
             vim_sf_id = self.vim.new_sf(name, sfi_id_list, sfc_encap=False)
 
             task["extra"]["created"] = True
+            task["extra"]["vim_status"] = "ACTIVE"
             task["error_msg"] = None
-            task["status"] = "DONE"
+            task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
             task["vim_id"] = vim_sf_id
             instance_element_update = {"status": "ACTIVE", "vim_sf_id": vim_sf_id, "error_msg": None}
-            return True, instance_element_update
+            return instance_element_update
 
         except (vimconn.vimconnException, VimThreadException) as e:
             self.logger.error("Error creating Service Function, task=%s: %s", task_id, str(e))
@@ -1186,24 +1152,24 @@ class vim_thread(threading.Thread):
             task["status"] = "FAILED"
             task["vim_id"] = None
             instance_element_update = {"status": "VIM_ERROR", "vim_sf_id": None, "error_msg": error_text}
-            return False, instance_element_update
+            return instance_element_update
 
     def del_sf(self, task):
         sf_vim_id = task["vim_id"]
         try:
             self.vim.delete_sf(sf_vim_id)
-            task["status"] = "DONE"
+            task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
             task["error_msg"] = None
-            return True, None
+            return None
 
         except vimconn.vimconnException as e:
             task["error_msg"] = self._format_vim_error_msg(str(e))
             if isinstance(e, vimconn.vimconnNotFoundException):
                 # If not found mark as Done and fill error_msg
-                task["status"] = "DONE"
-                return True, None
+                task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
+                return None
             task["status"] = "FAILED"
-            return False, None
+            return None
 
     def new_classification(self, task):
         vim_classification_id = None
@@ -1253,11 +1219,13 @@ class vim_thread(threading.Thread):
                 name, 'legacy_flow_classifier', definition)
 
             task["extra"]["created"] = True
+            task["extra"]["vim_status"] = "ACTIVE"
             task["error_msg"] = None
-            task["status"] = "DONE"
+            task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
             task["vim_id"] = vim_classification_id
-            instance_element_update = {"status": "ACTIVE", "vim_classification_id": vim_classification_id, "error_msg": None}
-            return True, instance_element_update
+            instance_element_update = {"status": "ACTIVE", "vim_classification_id": vim_classification_id,
+                                       "error_msg": None}
+            return instance_element_update
 
         except (vimconn.vimconnException, VimThreadException) as e:
             self.logger.error("Error creating Classification, task=%s: %s", task_id, str(e))
@@ -1266,31 +1234,31 @@ class vim_thread(threading.Thread):
             task["status"] = "FAILED"
             task["vim_id"] = None
             instance_element_update = {"status": "VIM_ERROR", "vim_classification_id": None, "error_msg": error_text}
-            return False, instance_element_update
+            return instance_element_update
 
     def del_classification(self, task):
         classification_vim_id = task["vim_id"]
         try:
             self.vim.delete_classification(classification_vim_id)
-            task["status"] = "DONE"
+            task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
             task["error_msg"] = None
-            return True, None
+            return None
 
         except vimconn.vimconnException as e:
             task["error_msg"] = self._format_vim_error_msg(str(e))
             if isinstance(e, vimconn.vimconnNotFoundException):
                 # If not found mark as Done and fill error_msg
-                task["status"] = "DONE"
-                return True, None
+                task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
+                return None
             task["status"] = "FAILED"
-            return False, None
+            return None
 
     def new_sfp(self, task):
         vim_sfp_id = None
         try:
-            params = task["params"]
             task_id = task["instance_action_id"] + "." + str(task["task_index"])
-            depending_tasks = [task.get("depends").get("TASK-" + str(tsk_id)) for tsk_id in task.get("extra").get("depends_on")]
+            depending_tasks = [task.get("depends").get("TASK-" + str(tsk_id)) for tsk_id in
+                               task.get("extra").get("depends_on")]
             error_text = ""
             sf_id_list = []
             classification_id_list = []
@@ -1307,11 +1275,12 @@ class vim_thread(threading.Thread):
             vim_sfp_id = self.vim.new_sfp(name, classification_id_list, sf_id_list, sfc_encap=False)
 
             task["extra"]["created"] = True
+            task["extra"]["vim_status"] = "ACTIVE"
             task["error_msg"] = None
-            task["status"] = "DONE"
+            task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
             task["vim_id"] = vim_sfp_id
             instance_element_update = {"status": "ACTIVE", "vim_sfp_id": vim_sfp_id, "error_msg": None}
-            return True, instance_element_update
+            return instance_element_update
 
         except (vimconn.vimconnException, VimThreadException) as e:
             self.logger.error("Error creating Service Function, task=%s: %s", task_id, str(e))
@@ -1320,22 +1289,21 @@ class vim_thread(threading.Thread):
             task["status"] = "FAILED"
             task["vim_id"] = None
             instance_element_update = {"status": "VIM_ERROR", "vim_sfp_id": None, "error_msg": error_text}
-            return False, instance_element_update
-        return
+            return instance_element_update
 
     def del_sfp(self, task):
         sfp_vim_id = task["vim_id"]
         try:
             self.vim.delete_sfp(sfp_vim_id)
-            task["status"] = "DONE"
+            task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
             task["error_msg"] = None
-            return True, None
+            return None
 
         except vimconn.vimconnException as e:
             task["error_msg"] = self._format_vim_error_msg(str(e))
             if isinstance(e, vimconn.vimconnNotFoundException):
                 # If not found mark as Done and fill error_msg
-                task["status"] = "DONE"
-                return True, None
+                task["status"] = "FINISHED"  # with FINISHED instead of DONE it will not be refreshing
+                return None
             task["status"] = "FAILED"
-            return False, None
+            return None
index a5da1dd..eecf10f 100644 (file)
@@ -331,7 +331,7 @@ class WimEngine(object):
     def derive_wan_link(self,
                         wim_usage,
                         instance_scenario_id, sce_net_id,
-                        networks, tenant):
+                        networks, tenant, related=None):
         """Create a instance_wim_nets record for the given information"""
         if sce_net_id in wim_usage:
             account_id = wim_usage[sce_net_id]
@@ -347,7 +347,8 @@ class WimEngine(object):
             'instance_scenario_id': instance_scenario_id,
             'sce_net_id': sce_net_id,
             'wim_id': wim_id,
-            'wim_account_id': account['uuid']
+            'wim_account_id': account['uuid'],
+            related: related
         }
 
     def derive_wan_links(self, wim_usage, networks, tenant=None):
@@ -366,6 +367,9 @@ class WimEngine(object):
             list: list of WAN links to be written to the database
         """
         # Group networks by key=(instance_scenario_id, sce_net_id)
+        related = None
+        if networks:
+            related = networks[0].get("related")
         filtered = _filter_multi_vim(networks)
         grouped_networks = _group_networks(filtered)
         datacenters_per_group = _count_datacenters(grouped_networks)
@@ -377,7 +381,7 @@ class WimEngine(object):
         # Keys are tuples(instance_scenario_id, sce_net_id)
         return [
             self.derive_wan_link(wim_usage,
-                                 key[0], key[1], grouped_networks[key], tenant)
+                                 key[0], key[1], grouped_networks[key], tenant, related)
             for key in wan_groups if wim_usage.get(key[1]) is not False
         ]