+ def _proccess_pending_tasks(self):
+ nb_created = 0
+ nb_processed = 0
+ while self.pending_tasks:
+ task = self.pending_tasks.pop(0)
+ nb_processed += 1
+ try:
+ # check if tasks that this depends on have been completed
+ dependency_not_completed = False
+ for task_index in task["extra"].get("depends_on", ()):
+ task_dependency = task["depends"].get("TASK-" + str(task_index))
+ if not task_dependency:
+ task_dependency = self._look_for_task(task["instance_action_id"], "TASK-" + str(task_index))
+ if not task_dependency:
+ raise VimThreadException(
+ "Cannot get depending net task trying to get depending task {}.{}".format(
+ task["instance_action_id"], task_index))
+ # task["depends"]["TASK-" + str(task_index)] = task_dependency #it references another object,so database must be look again
+ if task_dependency["status"] == "SCHEDULED":
+ dependency_not_completed = True
+ break
+ elif task_dependency["status"] == "FAILED":
+ raise VimThreadException(
+ "Cannot {} {}, (task {}.{}) because depends on failed {} {}, (task{}.{})".format(
+ task["action"], task["item"],
+ task["instance_action_id"], task["task_index"],
+ task_dependency["instance_action_id"], task_dependency["task_index"],
+ task_dependency["action"], task_dependency["item"]))
+ if dependency_not_completed:
+ # Move this task to the end.
+ task["extra"]["tries"] = task["extra"].get("tries", 0) + 1
+ if task["extra"]["tries"] <= 3:
+ self.pending_tasks.append(task)
+ continue
+ else:
+ raise VimThreadException(
+ "Cannot {} {}, (task {}.{}) because timeout waiting to complete {} {}, "
+ "(task {}.{})".format(task["action"], task["item"],
+ task["instance_action_id"], task["task_index"],
+ task_dependency["instance_action_id"], task_dependency["task_index"],
+ task_dependency["action"], task_dependency["item"]))
+
+ if task["status"] == "SUPERSEDED":
+ # not needed to do anything but update database with the new status
+ result = True
+ database_update = None
+ elif not self.vim:
+ task["status"] = "ERROR"
+ task["error_msg"] = self.error_status
+ result = False
+ database_update = {"status": "VIM_ERROR", "error_msg": task["error_msg"]}
+ elif task["item"] == 'instance_vms':
+ if task["action"] == "CREATE":
+ result, database_update = self.new_vm(task)
+ nb_created += 1
+ elif task["action"] == "DELETE":
+ result, database_update = self.del_vm(task)
+ else:
+ raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
+ elif task["item"] == 'instance_nets':
+ if task["action"] == "CREATE":
+ result, database_update = self.new_net(task)
+ nb_created += 1
+ elif task["action"] == "DELETE":
+ result, database_update = self.del_net(task)
+ elif task["action"] == "FIND":
+ result, database_update = self.get_net(task)
+ else:
+ raise vimconn.vimconnException(self.name + "unknown task action {}".format(task["action"]))
+ else:
+ raise vimconn.vimconnException(self.name + "unknown task item {}".format(task["item"]))
+ # TODO
+ except VimThreadException as e:
+ result = False
+ task["error_msg"] = str(e)
+ task["status"] = "FAILED"
+ database_update = {"status": "VIM_ERROR", "error_msg": task["error_msg"]}
+ if task["item"] == 'instance_vms':
+ database_update["vim_vm_id"] = None
+ elif task["item"] == 'instance_nets':
+ database_update["vim_net_id"] = None
+
+ if task["action"] == "DELETE":
+ action_key = task["item"] + task["item_id"]
+ del self.grouped_tasks[action_key]
+ elif task["action"] in ("CREATE", "FIND") and task["status"] in ("DONE", "BUILD"):
+ self._insert_refresh(task)
+
+ task_id = task["instance_action_id"] + "." + str(task["task_index"])
+ self.logger.debug("task={} item={} action={} result={}:'{}' params={}".format(
+ task_id, task["item"], task["action"], task["status"],
+ task["vim_id"] if task["status"] == "DONE" else task.get("error_msg"), task["params"]))
+ try:
+ now = time.time()
+ with self.db_lock:
+ self.db.update_rows(
+ table="vim_actions",
+ UPDATE={"status": task["status"], "vim_id": task.get("vim_id"), "modified_at": now,
+ "error_msg": task["error_msg"],
+ "extra": yaml.safe_dump(task["extra"], default_flow_style=True, width=256)},
+ WHERE={"instance_action_id": task["instance_action_id"], "task_index": task["task_index"]})
+ if result is not None:
+ self.db.update_rows(
+ table="instance_actions",
+ UPDATE={("number_done" if result else "number_failed"): {"INCREMENT": 1},
+ "modified_at": now},
+ WHERE={"uuid": task["instance_action_id"]})
+ if database_update:
+ self.db.update_rows(table=task["item"],
+ UPDATE=database_update,
+ WHERE={"uuid": task["item_id"]})
+ except db_base_Exception as e:
+ self.logger.error("task={} Error updating database {}".format(task_id, e), exc_info=True)
+
+ if nb_created == 10:
+ break
+ return nb_processed
+
+ def _insert_pending_tasks(self, vim_actions_list):
+ for task in vim_actions_list:
+ if task["datacenter_vim_id"] != self.datacenter_tenant_id:
+ continue
+ item = task["item"]
+ item_id = task["item_id"]
+ action_key = item + item_id
+ if action_key not in self.grouped_tasks:
+ self.grouped_tasks[action_key] = []
+ task["params"] = None
+ task["depends"] = {}
+ if task["extra"]:
+ extra = yaml.load(task["extra"])
+ task["extra"] = extra
+ task["params"] = extra.get("params")
+ depends_on_list = extra.get("depends_on")
+ if depends_on_list:
+ for index in depends_on_list:
+ if index < len(vim_actions_list) and vim_actions_list[index]["task_index"] == index and\
+ vim_actions_list[index]["instance_action_id"] == task["instance_action_id"]:
+ task["depends"]["TASK-" + str(index)] = vim_actions_list[index]
+ if extra.get("interfaces"):
+ task["vim_interfaces"] = {}
+ else:
+ task["extra"] = {}
+ if "error_msg" not in task:
+ task["error_msg"] = None
+ if "vim_id" not in task:
+ task["vim_id"] = None
+
+ if task["action"] == "DELETE":
+ need_delete_action = False
+ for to_supersede in self.grouped_tasks.get(action_key, ()):
+ if to_supersede["action"] == "FIND" and to_supersede.get("vim_id"):
+ task["vim_id"] = to_supersede["vim_id"]
+ if to_supersede["action"] == "CREATE" and to_supersede["extra"].get("created", True) and \
+ (to_supersede.get("vim_id") or to_supersede["extra"].get("sdn_net_id")):
+ need_delete_action = True
+ task["vim_id"] = to_supersede["vim_id"]
+ if to_supersede["extra"].get("sdn_net_id"):
+ task["extra"]["sdn_net_id"] = to_supersede["extra"]["sdn_net_id"]
+ if to_supersede["extra"].get("interfaces"):
+ task["extra"]["interfaces"] = to_supersede["extra"]["interfaces"]
+ if to_supersede["extra"].get("created_items"):
+ if not task["extra"].get("created_items"):
+ task["extra"]["created_items"] = {}
+ task["extra"]["created_items"].update(to_supersede["extra"]["created_items"])
+ # Mark task as SUPERSEDED.
+ # If task is in self.pending_tasks, it will be removed and database will be update
+ # If task is in self.refresh_tasks, it will be removed
+ to_supersede["status"] = "SUPERSEDED"
+ if not need_delete_action:
+ task["status"] = "SUPERSEDED"
+
+ self.grouped_tasks[action_key].append(task)
+ self.pending_tasks.append(task)
+ elif task["status"] == "SCHEDULED":
+ self.grouped_tasks[action_key].append(task)
+ self.pending_tasks.append(task)
+ elif task["action"] in ("CREATE", "FIND"):
+ self.grouped_tasks[action_key].append(task)
+ if task["status"] in ("DONE", "BUILD"):
+ self._insert_refresh(task)
+ # TODO add VM reset, get console, etc...
+ else:
+ raise vimconn.vimconnException(self.name + "unknown vim_action action {}".format(task["action"]))
+