from shutil import rmtree
import threading
import time
+import traceback
from unittest.mock import Mock
from importlib_metadata import entry_points
"created": created,
"created_items": created_items,
"vim_details": None,
+ "vim_message": None,
}
self.logger.debug(
"task={} {} new-net={} created={}".format(
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
ro_vim_item_update["vim_name"] = vim_info.get("name")
if vim_info["status"] in ("ERROR", "VIM_ERROR"):
- if ro_task["vim_info"]["vim_details"] != vim_info.get("error_msg"):
- ro_vim_item_update["vim_details"] = vim_info.get("error_msg")
+ if ro_task["vim_info"]["vim_message"] != vim_info.get("error_msg"):
+ ro_vim_item_update["vim_message"] = vim_info.get("error_msg")
elif vim_info["status"] == "DELETED":
ro_vim_item_update["vim_id"] = None
- ro_vim_item_update["vim_details"] = "Deleted externally"
+ ro_vim_item_update["vim_message"] = "Deleted externally"
else:
if ro_task["vim_info"]["vim_details"] != vim_info["vim_info"]:
ro_vim_item_update["vim_details"] = vim_info["vim_info"]
ro_task["target_id"],
vim_id,
ro_vim_item_update.get("vim_status"),
- ro_vim_item_update.get("vim_details")
+ ro_vim_item_update.get("vim_message")
if ro_vim_item_update.get("vim_status") != "ACTIVE"
else "",
)
ro_vim_item_update_ok = {
"vim_status": "DELETED",
"created": False,
- "vim_details": "DELETED",
+ "vim_message": "DELETED",
"vim_id": None,
}
net_vim_id, ro_task["vim_info"]["created_items"]
)
except vimconn.VimConnNotFoundException:
- ro_vim_item_update_ok["vim_details"] = "already deleted"
+ ro_vim_item_update_ok["vim_message"] = "already deleted"
except vimconn.VimConnException as e:
self.logger.error(
"ro_task={} vim={} del-net={}: {}".format(
)
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
- "vim_details": "Error while deleting: {}".format(e),
+ "vim_message": "Error while deleting: {}".format(e),
}
return "FAILED", ro_vim_item_update
task_id,
ro_task["target_id"],
net_vim_id,
- ro_vim_item_update_ok.get("vim_details", ""),
+ ro_vim_item_update_ok.get("vim_message", ""),
)
)
vim_vm_id, created_items = target_vim.new_vminstance(**params_copy)
interfaces = [iface["vim_id"] for iface in params_copy["net_list"]]
+ # add to created items previous_created_volumes (healing)
+ if task.get("previous_created_volumes"):
+ for k, v in task["previous_created_volumes"].items():
+ created_items[k] = v
+
ro_vim_item_update = {
"vim_id": vim_vm_id,
"vim_status": "BUILD",
"created": created,
"created_items": created_items,
"vim_details": None,
+ "vim_message": None,
"interfaces_vim_ids": interfaces,
"interfaces": [],
+ "interfaces_backup": [],
}
self.logger.debug(
"task={} {} new-vm={} created={}".format(
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
ro_vim_item_update_ok = {
"vim_status": "DELETED",
"created": False,
- "vim_details": "DELETED",
+ "vim_message": "DELETED",
"vim_id": None,
}
try:
+ self.logger.debug(
+ "delete_vminstance: vm_vim_id={} created_items={}".format(
+ vm_vim_id, ro_task["vim_info"]["created_items"]
+ )
+ )
if vm_vim_id or ro_task["vim_info"]["created_items"]:
target_vim = self.my_vims[ro_task["target_id"]]
target_vim.delete_vminstance(
- vm_vim_id, ro_task["vim_info"]["created_items"]
+ vm_vim_id,
+ ro_task["vim_info"]["created_items"],
+ ro_task["vim_info"].get("volumes_to_hold", []),
)
except vimconn.VimConnNotFoundException:
- ro_vim_item_update_ok["vim_details"] = "already deleted"
+ ro_vim_item_update_ok["vim_message"] = "already deleted"
except vimconn.VimConnException as e:
self.logger.error(
"ro_task={} vim={} del-vm={}: {}".format(
)
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
- "vim_details": "Error while deleting: {}".format(e),
+ "vim_message": "Error while deleting: {}".format(e),
}
return "FAILED", ro_vim_item_update
task_id,
ro_task["target_id"],
vm_vim_id,
- ro_vim_item_update_ok.get("vim_details", ""),
+ ro_vim_item_update_ok.get("vim_message", ""),
)
)
ro_vim_item_update["vim_name"] = vim_info.get("name")
if vim_info["status"] in ("ERROR", "VIM_ERROR"):
- if ro_task["vim_info"]["vim_details"] != vim_info.get("error_msg"):
- ro_vim_item_update["vim_details"] = vim_info.get("error_msg")
+ if ro_task["vim_info"]["vim_message"] != vim_info.get("error_msg"):
+ ro_vim_item_update["vim_message"] = vim_info.get("error_msg")
elif vim_info["status"] == "DELETED":
ro_vim_item_update["vim_id"] = None
- ro_vim_item_update["vim_details"] = "Deleted externally"
+ ro_vim_item_update["vim_message"] = "Deleted externally"
else:
if ro_task["vim_info"]["vim_details"] != vim_info["vim_info"]:
ro_vim_item_update["vim_details"] = vim_info["vim_info"]
ro_task["target_id"],
vim_id,
ro_vim_item_update.get("vim_status"),
- ro_vim_item_update.get("vim_details")
+ ro_vim_item_update.get("vim_message")
if ro_vim_item_update.get("vim_status") != "ACTIVE"
else "",
)
except (vimconn.VimConnException, NsWorkerException) as e:
retries += 1
+ self.logger.debug(traceback.format_exc())
if retries < self.max_retries_inject_ssh_key:
return (
"BUILD",
self.logger.error(
"task={} {} inject-ssh-key: {}".format(task_id, ro_task["target_id"], e)
)
- ro_vim_item_update = {"vim_details": str(e)}
+ ro_vim_item_update = {"vim_message": str(e)}
return "FAILED", ro_vim_item_update, db_task_update
"created": created,
"created_items": created_items,
"vim_details": None,
+ "vim_message": None,
}
self.logger.debug(
"task={} {} new-image={} created={}".format(
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
ro_vim_item_update_ok = {
"vim_status": "DELETED",
"created": False,
- "vim_details": "DELETED",
+ "vim_message": "DELETED",
"vim_id": None,
}
target_vim = self.my_vims[ro_task["target_id"]]
target_vim.delete_flavor(flavor_vim_id)
except vimconn.VimConnNotFoundException:
- ro_vim_item_update_ok["vim_details"] = "already deleted"
+ ro_vim_item_update_ok["vim_message"] = "already deleted"
except vimconn.VimConnException as e:
self.logger.error(
"ro_task={} vim={} del-flavor={}: {}".format(
)
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
- "vim_details": "Error while deleting: {}".format(e),
+ "vim_message": "Error while deleting: {}".format(e),
}
return "FAILED", ro_vim_item_update
task_id,
ro_task["target_id"],
flavor_vim_id,
- ro_vim_item_update_ok.get("vim_details", ""),
+ ro_vim_item_update_ok.get("vim_message", ""),
)
)
"created": created,
"created_items": created_items,
"vim_details": None,
+ "vim_message": None,
}
self.logger.debug(
"task={} {} new-flavor={} created={}".format(
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
ro_vim_item_update_ok = {
"vim_status": "DELETED",
"created": False,
- "vim_details": "DELETED",
+ "vim_message": "DELETED",
"vim_id": None,
}
target_vim = self.my_vims[ro_task["target_id"]]
target_vim.delete_affinity_group(affinity_group_vim_id)
except vimconn.VimConnNotFoundException:
- ro_vim_item_update_ok["vim_details"] = "already deleted"
+ ro_vim_item_update_ok["vim_message"] = "already deleted"
except vimconn.VimConnException as e:
self.logger.error(
"ro_task={} vim={} del-affinity-or-anti-affinity-group={}: {}".format(
)
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
- "vim_details": "Error while deleting: {}".format(e),
+ "vim_message": "Error while deleting: {}".format(e),
}
return "FAILED", ro_vim_item_update
task_id,
ro_task["target_id"],
affinity_group_vim_id,
- ro_vim_item_update_ok.get("vim_details", ""),
+ ro_vim_item_update_ok.get("vim_message", ""),
)
)
try:
affinity_group_vim_id = None
+ affinity_group_data = None
if task.get("params"):
- affinity_group_data = task["params"]["affinity_group_data"]
+ affinity_group_data = task["params"].get("affinity_group_data")
+
+ if affinity_group_data and affinity_group_data.get("vim-affinity-group-id"):
+ try:
+ param_affinity_group_id = task["params"]["affinity_group_data"].get(
+ "vim-affinity-group-id"
+ )
+ affinity_group_vim_id = target_vim.get_affinity_group(
+ param_affinity_group_id
+ ).get("id")
+ except vimconn.VimConnNotFoundException:
+ self.logger.error(
+ "task={} {} new-affinity-or-anti-affinity-group. Provided VIM Affinity Group ID {}"
+ "could not be found at VIM. Creating a new one.".format(
+ task_id, ro_task["target_id"], param_affinity_group_id
+ )
+ )
+
+ if not affinity_group_vim_id and affinity_group_data:
affinity_group_vim_id = target_vim.new_affinity_group(
affinity_group_data
)
"created": created,
"created_items": created_items,
"vim_details": None,
+ "vim_message": None,
}
self.logger.debug(
"task={} {} new-affinity-or-anti-affinity-group={} created={}".format(
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
+class VimInteractionUpdateVdu(VimInteractionBase):
+ def exec(self, ro_task, task_index, task_depends):
+ task = ro_task["tasks"][task_index]
+ task_id = task["task_id"]
+ db_task_update = {"retries": 0}
+ created = False
+ created_items = {}
+ target_vim = self.my_vims[ro_task["target_id"]]
+
+ try:
+ if task.get("params"):
+ vim_vm_id = task["params"].get("vim_vm_id")
+ action = task["params"].get("action")
+ context = {action: action}
+ target_vim.action_vminstance(vim_vm_id, context)
+ # created = True
+ ro_vim_item_update = {
+ "vim_id": vim_vm_id,
+ "vim_status": "DONE",
+ "created": created,
+ "created_items": created_items,
+ "vim_details": None,
+ "vim_message": None,
+ }
+ self.logger.debug(
+ "task={} {} vm-migration done".format(task_id, ro_task["target_id"])
+ )
+ return "DONE", ro_vim_item_update, db_task_update
+ except (vimconn.VimConnException, NsWorkerException) as e:
+ self.logger.error(
+ "task={} vim={} VM Migration:"
+ " {}".format(task_id, ro_task["target_id"], e)
+ )
+ ro_vim_item_update = {
+ "vim_status": "VIM_ERROR",
+ "created": created,
+ "vim_message": str(e),
+ }
+
+ return "FAILED", ro_vim_item_update, db_task_update
+
+
class VimInteractionSdnNet(VimInteractionBase):
@staticmethod
def _match_pci(port_pci, mapping):
"created_items": created_items,
"connected_ports": connected_ports,
"vim_details": sdn_info,
+ "vim_message": None,
"last_update": last_update,
}
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
ro_vim_item_update_ok = {
"vim_status": "DELETED",
"created": False,
- "vim_details": "DELETED",
+ "vim_message": "DELETED",
"vim_id": None,
}
isinstance(e, sdnconn.SdnConnectorError)
and e.http_code == HTTPStatus.NOT_FOUND.value
):
- ro_vim_item_update_ok["vim_details"] = "already deleted"
+ ro_vim_item_update_ok["vim_message"] = "already deleted"
else:
self.logger.error(
"ro_task={} vim={} del-sdn-net={}: {}".format(
)
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
- "vim_details": "Error while deleting: {}".format(e),
+ "vim_message": "Error while deleting: {}".format(e),
}
return "FAILED", ro_vim_item_update
task_id,
ro_task["target_id"],
sdn_vim_id,
- ro_vim_item_update_ok.get("vim_details", ""),
+ ro_vim_item_update_ok.get("vim_message", ""),
)
)
return "DONE", ro_vim_item_update_ok
+class VimInteractionMigration(VimInteractionBase):
+ def exec(self, ro_task, task_index, task_depends):
+ task = ro_task["tasks"][task_index]
+ task_id = task["task_id"]
+ db_task_update = {"retries": 0}
+ target_vim = self.my_vims[ro_task["target_id"]]
+ vim_interfaces = []
+ created = False
+ created_items = {}
+ refreshed_vim_info = {}
+
+ try:
+ if task.get("params"):
+ vim_vm_id = task["params"].get("vim_vm_id")
+ migrate_host = task["params"].get("migrate_host")
+ _, migrated_compute_node = target_vim.migrate_instance(
+ vim_vm_id, migrate_host
+ )
+
+ if migrated_compute_node:
+ # When VM is migrated, vdu["vim_info"] needs to be updated
+ vdu_old_vim_info = task["params"]["vdu_vim_info"].get(
+ ro_task["target_id"]
+ )
+
+ # Refresh VM to get new vim_info
+ vm_to_refresh_list = [vim_vm_id]
+ vim_dict = target_vim.refresh_vms_status(vm_to_refresh_list)
+ refreshed_vim_info = vim_dict[vim_vm_id]
+
+ if refreshed_vim_info.get("interfaces"):
+ for old_iface in vdu_old_vim_info.get("interfaces"):
+ iface = next(
+ (
+ iface
+ for iface in refreshed_vim_info["interfaces"]
+ if old_iface["vim_interface_id"]
+ == iface["vim_interface_id"]
+ ),
+ None,
+ )
+ vim_interfaces.append(iface)
+
+ ro_vim_item_update = {
+ "vim_id": vim_vm_id,
+ "vim_status": "ACTIVE",
+ "created": created,
+ "created_items": created_items,
+ "vim_details": None,
+ "vim_message": None,
+ }
+
+ if refreshed_vim_info and refreshed_vim_info.get("status") not in (
+ "ERROR",
+ "VIM_ERROR",
+ ):
+ ro_vim_item_update["vim_details"] = refreshed_vim_info["vim_info"]
+
+ if vim_interfaces:
+ ro_vim_item_update["interfaces"] = vim_interfaces
+
+ self.logger.debug(
+ "task={} {} vm-migration done".format(task_id, ro_task["target_id"])
+ )
+
+ return "DONE", ro_vim_item_update, db_task_update
+
+ except (vimconn.VimConnException, NsWorkerException) as e:
+ self.logger.error(
+ "task={} vim={} VM Migration:"
+ " {}".format(task_id, ro_task["target_id"], e)
+ )
+ ro_vim_item_update = {
+ "vim_status": "VIM_ERROR",
+ "created": created,
+ "vim_message": str(e),
+ }
+
+ return "FAILED", ro_vim_item_update, db_task_update
+
+
class NsWorker(threading.Thread):
REFRESH_BUILD = 5 # 5 seconds
REFRESH_ACTIVE = 60 # 1 minute
"sdn_net": VimInteractionSdnNet(
self.db, self.my_vims, self.db_vims, self.logger
),
+ "update": VimInteractionUpdateVdu(
+ self.db, self.my_vims, self.db_vims, self.logger
+ ),
"affinity-or-anti-affinity-group": VimInteractionAffinityGroup(
self.db, self.my_vims, self.db_vims, self.logger
),
+ "migrate": VimInteractionMigration(
+ self.db, self.my_vims, self.db_vims, self.logger
+ ),
}
self.time_last_task_processed = None
# lists of tasks to delete because nsrs or vnfrs has been deleted from db
1642158640.7986135;1642158640.7986135;1642158640.7986135;b134c9494e75:0a
;vim:b7ff9e24-8868-4d68-8a57-a59dc11d0327;None;{'created': False,
'created_items': None, 'vim_id': None, 'vim_name': None, 'vim_status': None,
- 'vim_details': None, 'refresh_at': None};1;SCHEDULED;
+ 'vim_details': None, 'vim_message': None, 'refresh_at': None};1;SCHEDULED;
888f1864-749a-4fc2-bc1a-97c0fffd6a6f;0;888f1864-749a-4fc2-bc1a-97c0fffd6a6f:2;
CREATE;image;{'filter_dict': {'name': 'ubuntu-os-cloud:image-family:ubuntu-1804-lts'}}
"""
"created_items", False
)
+ self.logger.warning("Needed delete: {}".format(needed_delete))
if my_task["status"] == "FAILED":
return None, None # TODO need to be retry??
needed_delete = False
if needed_delete:
+ self.logger.warning(
+ "Deleting ro_task={} task_index={}".format(ro_task, task_index)
+ )
return self.item2class[my_task["item"]].delete(ro_task, task_index)
else:
return "SUPERSEDED", None
exc_info=True,
)
- return "FAILED", {"vim_status": "VIM_ERROR", "vim_details": str(e)}
+ return "FAILED", {"vim_status": "VIM_ERROR", "vim_message": str(e)}
def _create_task(self, ro_task, task_index, task_depends, db_update):
"""
)
task_status = "FAILED"
- ro_vim_item_update = {"vim_status": "VIM_ERROR", "vim_details": str(e)}
+ ro_vim_item_update = {"vim_status": "VIM_ERROR", "vim_message": str(e)}
# TODO update ro_vim_item_update
return task_status, ro_vim_item_update
fail_on_empty=False,
)
+ self.logger.warning("ro_task_dependency={}".format(ro_task_dependency))
if ro_task_dependency:
- for task_index, task in ro_task_dependency["tasks"]:
+ for task_index, task in enumerate(ro_task_dependency["tasks"]):
if task["task_id"] == task_id:
return ro_task_dependency, task_index
raise NsWorkerException("Cannot get depending task {}".format(task_id))
dependency_task = dependency_ro_task["tasks"][
dependency_task_index
]
+ self.logger.warning(
+ "dependency_ro_task={} dependency_task_index={}".format(
+ dependency_ro_task, dependency_task_index
+ )
+ )
if dependency_task["status"] == "SCHEDULED":
dependency_not_completed = True
dependency_task["item"],
dependency_task_id,
dependency_ro_task["vim_info"].get(
- "vim_details"
+ "vim_message"
),
)
self.logger.error(
] = dependency_ro_task["vim_info"]["vim_id"]
if dependency_not_completed:
+ self.logger.warning(
+ "DEPENDENCY NOT COMPLETED {}".format(
+ dependency_ro_task["vim_info"]["vim_id"]
+ )
+ )
# TODO set at vim_info.vim_details that it is waiting
continue
new_status = "FAILED"
db_vim_info_update = {
"vim_status": "VIM_ERROR",
- "vim_details": str(e),
+ "vim_message": str(e),
}
if not isinstance(
path_vim_status + "." + k: v
for k, v in ro_vim_item_update.items()
if k
- in ("vim_id", "vim_details", "vim_name", "vim_status", "interfaces")
+ in (
+ "vim_id",
+ "vim_details",
+ "vim_message",
+ "vim_name",
+ "vim_status",
+ "interfaces",
+ "interfaces_backup",
+ )
}
if path_vim_status.startswith("vdur."):
).split(";")[0]
self.db.set_one(table, q_filter={"_id": _id}, update_dict=update_dict)
+
+ # If interfaces exists, it backups VDU interfaces in the DB for healing operations
+ if ro_vim_item_update.get("interfaces"):
+ search_key = path_vim_status + ".interfaces"
+ if update_dict.get(search_key):
+ interfaces_backup_update = {
+ path_vim_status + ".interfaces_backup": update_dict[search_key]
+ }
+
+ self.db.set_one(
+ table,
+ q_filter={"_id": _id},
+ update_dict=interfaces_backup_update,
+ )
+
else:
update_dict = {path_item + ".status": "DELETED"}
self.db.set_one(
"""
ro_task = self._get_db_task()
if ro_task:
+ self.logger.warning("Task to process: {}".format(ro_task))
+ time.sleep(1)
self._process_pending_tasks(ro_task)
busy = True
if not busy:
projects / osm / RO.git / blobdiff