Feature 10910: Migration of Openstack based VM instances
[osm/RO.git] / NG-RO / osm_ng_ro / ns_thread.py
index 51d206e..617f58b 100644 (file)
@@ -32,6 +32,7 @@ import queue
 from shutil import rmtree
 import threading
 import time
+import traceback
 from unittest.mock import Mock
 
 from importlib_metadata import entry_points
@@ -210,6 +211,7 @@ class VimInteractionNet(VimInteractionBase):
                 "created": created,
                 "created_items": created_items,
                 "vim_details": None,
+                "vim_message": None,
             }
             self.logger.debug(
                 "task={} {} new-net={} created={}".format(
@@ -225,7 +227,7 @@ class VimInteractionNet(VimInteractionBase):
             ro_vim_item_update = {
                 "vim_status": "VIM_ERROR",
                 "created": created,
-                "vim_details": str(e),
+                "vim_message": str(e),
             }
 
             return "FAILED", ro_vim_item_update
@@ -265,11 +267,11 @@ class VimInteractionNet(VimInteractionBase):
             ro_vim_item_update["vim_name"] = vim_info.get("name")
 
         if vim_info["status"] in ("ERROR", "VIM_ERROR"):
-            if ro_task["vim_info"]["vim_details"] != vim_info.get("error_msg"):
-                ro_vim_item_update["vim_details"] = vim_info.get("error_msg")
+            if ro_task["vim_info"]["vim_message"] != vim_info.get("error_msg"):
+                ro_vim_item_update["vim_message"] = vim_info.get("error_msg")
         elif vim_info["status"] == "DELETED":
             ro_vim_item_update["vim_id"] = None
-            ro_vim_item_update["vim_details"] = "Deleted externally"
+            ro_vim_item_update["vim_message"] = "Deleted externally"
         else:
             if ro_task["vim_info"]["vim_details"] != vim_info["vim_info"]:
                 ro_vim_item_update["vim_details"] = vim_info["vim_info"]
@@ -281,7 +283,7 @@ class VimInteractionNet(VimInteractionBase):
                     ro_task["target_id"],
                     vim_id,
                     ro_vim_item_update.get("vim_status"),
-                    ro_vim_item_update.get("vim_details")
+                    ro_vim_item_update.get("vim_message")
                     if ro_vim_item_update.get("vim_status") != "ACTIVE"
                     else "",
                 )
@@ -296,7 +298,7 @@ class VimInteractionNet(VimInteractionBase):
         ro_vim_item_update_ok = {
             "vim_status": "DELETED",
             "created": False,
-            "vim_details": "DELETED",
+            "vim_message": "DELETED",
             "vim_id": None,
         }
 
@@ -307,7 +309,7 @@ class VimInteractionNet(VimInteractionBase):
                     net_vim_id, ro_task["vim_info"]["created_items"]
                 )
         except vimconn.VimConnNotFoundException:
-            ro_vim_item_update_ok["vim_details"] = "already deleted"
+            ro_vim_item_update_ok["vim_message"] = "already deleted"
         except vimconn.VimConnException as e:
             self.logger.error(
                 "ro_task={} vim={} del-net={}: {}".format(
@@ -316,7 +318,7 @@ class VimInteractionNet(VimInteractionBase):
             )
             ro_vim_item_update = {
                 "vim_status": "VIM_ERROR",
-                "vim_details": "Error while deleting: {}".format(e),
+                "vim_message": "Error while deleting: {}".format(e),
             }
 
             return "FAILED", ro_vim_item_update
@@ -326,7 +328,7 @@ class VimInteractionNet(VimInteractionBase):
                 task_id,
                 ro_task["target_id"],
                 net_vim_id,
-                ro_vim_item_update_ok.get("vim_details", ""),
+                ro_vim_item_update_ok.get("vim_message", ""),
             )
         )
 
@@ -395,6 +397,7 @@ class VimInteractionVdu(VimInteractionBase):
                 "created": created,
                 "created_items": created_items,
                 "vim_details": None,
+                "vim_message": None,
                 "interfaces_vim_ids": interfaces,
                 "interfaces": [],
             }
@@ -412,7 +415,7 @@ class VimInteractionVdu(VimInteractionBase):
             ro_vim_item_update = {
                 "vim_status": "VIM_ERROR",
                 "created": created,
-                "vim_details": str(e),
+                "vim_message": str(e),
             }
 
             return "FAILED", ro_vim_item_update
@@ -424,7 +427,7 @@ class VimInteractionVdu(VimInteractionBase):
         ro_vim_item_update_ok = {
             "vim_status": "DELETED",
             "created": False,
-            "vim_details": "DELETED",
+            "vim_message": "DELETED",
             "vim_id": None,
         }
 
@@ -442,7 +445,7 @@ class VimInteractionVdu(VimInteractionBase):
                     ro_task["vim_info"].get("volumes_to_hold", []),
                 )
         except vimconn.VimConnNotFoundException:
-            ro_vim_item_update_ok["vim_details"] = "already deleted"
+            ro_vim_item_update_ok["vim_message"] = "already deleted"
         except vimconn.VimConnException as e:
             self.logger.error(
                 "ro_task={} vim={} del-vm={}: {}".format(
@@ -451,7 +454,7 @@ class VimInteractionVdu(VimInteractionBase):
             )
             ro_vim_item_update = {
                 "vim_status": "VIM_ERROR",
-                "vim_details": "Error while deleting: {}".format(e),
+                "vim_message": "Error while deleting: {}".format(e),
             }
 
             return "FAILED", ro_vim_item_update
@@ -461,7 +464,7 @@ class VimInteractionVdu(VimInteractionBase):
                 task_id,
                 ro_task["target_id"],
                 vm_vim_id,
-                ro_vim_item_update_ok.get("vim_details", ""),
+                ro_vim_item_update_ok.get("vim_message", ""),
             )
         )
 
@@ -549,11 +552,11 @@ class VimInteractionVdu(VimInteractionBase):
             ro_vim_item_update["vim_name"] = vim_info.get("name")
 
         if vim_info["status"] in ("ERROR", "VIM_ERROR"):
-            if ro_task["vim_info"]["vim_details"] != vim_info.get("error_msg"):
-                ro_vim_item_update["vim_details"] = vim_info.get("error_msg")
+            if ro_task["vim_info"]["vim_message"] != vim_info.get("error_msg"):
+                ro_vim_item_update["vim_message"] = vim_info.get("error_msg")
         elif vim_info["status"] == "DELETED":
             ro_vim_item_update["vim_id"] = None
-            ro_vim_item_update["vim_details"] = "Deleted externally"
+            ro_vim_item_update["vim_message"] = "Deleted externally"
         else:
             if ro_task["vim_info"]["vim_details"] != vim_info["vim_info"]:
                 ro_vim_item_update["vim_details"] = vim_info["vim_info"]
@@ -565,7 +568,7 @@ class VimInteractionVdu(VimInteractionBase):
                     ro_task["target_id"],
                     vim_id,
                     ro_vim_item_update.get("vim_status"),
-                    ro_vim_item_update.get("vim_details")
+                    ro_vim_item_update.get("vim_message")
                     if ro_vim_item_update.get("vim_status") != "ACTIVE"
                     else "",
                 )
@@ -602,6 +605,7 @@ class VimInteractionVdu(VimInteractionBase):
         except (vimconn.VimConnException, NsWorkerException) as e:
             retries += 1
 
+            self.logger.debug(traceback.format_exc())
             if retries < self.max_retries_inject_ssh_key:
                 return (
                     "BUILD",
@@ -615,7 +619,7 @@ class VimInteractionVdu(VimInteractionBase):
             self.logger.error(
                 "task={} {} inject-ssh-key: {}".format(task_id, ro_task["target_id"], e)
             )
-            ro_vim_item_update = {"vim_details": str(e)}
+            ro_vim_item_update = {"vim_message": str(e)}
 
             return "FAILED", ro_vim_item_update, db_task_update
 
@@ -654,6 +658,7 @@ class VimInteractionImage(VimInteractionBase):
                 "created": created,
                 "created_items": created_items,
                 "vim_details": None,
+                "vim_message": None,
             }
             self.logger.debug(
                 "task={} {} new-image={} created={}".format(
@@ -669,7 +674,7 @@ class VimInteractionImage(VimInteractionBase):
             ro_vim_item_update = {
                 "vim_status": "VIM_ERROR",
                 "created": created,
-                "vim_details": str(e),
+                "vim_message": str(e),
             }
 
             return "FAILED", ro_vim_item_update
@@ -683,7 +688,7 @@ class VimInteractionFlavor(VimInteractionBase):
         ro_vim_item_update_ok = {
             "vim_status": "DELETED",
             "created": False,
-            "vim_details": "DELETED",
+            "vim_message": "DELETED",
             "vim_id": None,
         }
 
@@ -692,7 +697,7 @@ class VimInteractionFlavor(VimInteractionBase):
                 target_vim = self.my_vims[ro_task["target_id"]]
                 target_vim.delete_flavor(flavor_vim_id)
         except vimconn.VimConnNotFoundException:
-            ro_vim_item_update_ok["vim_details"] = "already deleted"
+            ro_vim_item_update_ok["vim_message"] = "already deleted"
         except vimconn.VimConnException as e:
             self.logger.error(
                 "ro_task={} vim={} del-flavor={}: {}".format(
@@ -701,7 +706,7 @@ class VimInteractionFlavor(VimInteractionBase):
             )
             ro_vim_item_update = {
                 "vim_status": "VIM_ERROR",
-                "vim_details": "Error while deleting: {}".format(e),
+                "vim_message": "Error while deleting: {}".format(e),
             }
 
             return "FAILED", ro_vim_item_update
@@ -711,7 +716,7 @@ class VimInteractionFlavor(VimInteractionBase):
                 task_id,
                 ro_task["target_id"],
                 flavor_vim_id,
-                ro_vim_item_update_ok.get("vim_details", ""),
+                ro_vim_item_update_ok.get("vim_message", ""),
             )
         )
 
@@ -747,6 +752,7 @@ class VimInteractionFlavor(VimInteractionBase):
                 "created": created,
                 "created_items": created_items,
                 "vim_details": None,
+                "vim_message": None,
             }
             self.logger.debug(
                 "task={} {} new-flavor={} created={}".format(
@@ -762,7 +768,7 @@ class VimInteractionFlavor(VimInteractionBase):
             ro_vim_item_update = {
                 "vim_status": "VIM_ERROR",
                 "created": created,
-                "vim_details": str(e),
+                "vim_message": str(e),
             }
 
             return "FAILED", ro_vim_item_update
@@ -776,7 +782,7 @@ class VimInteractionAffinityGroup(VimInteractionBase):
         ro_vim_item_update_ok = {
             "vim_status": "DELETED",
             "created": False,
-            "vim_details": "DELETED",
+            "vim_message": "DELETED",
             "vim_id": None,
         }
 
@@ -785,7 +791,7 @@ class VimInteractionAffinityGroup(VimInteractionBase):
                 target_vim = self.my_vims[ro_task["target_id"]]
                 target_vim.delete_affinity_group(affinity_group_vim_id)
         except vimconn.VimConnNotFoundException:
-            ro_vim_item_update_ok["vim_details"] = "already deleted"
+            ro_vim_item_update_ok["vim_message"] = "already deleted"
         except vimconn.VimConnException as e:
             self.logger.error(
                 "ro_task={} vim={} del-affinity-or-anti-affinity-group={}: {}".format(
@@ -794,7 +800,7 @@ class VimInteractionAffinityGroup(VimInteractionBase):
             )
             ro_vim_item_update = {
                 "vim_status": "VIM_ERROR",
-                "vim_details": "Error while deleting: {}".format(e),
+                "vim_message": "Error while deleting: {}".format(e),
             }
 
             return "FAILED", ro_vim_item_update
@@ -804,7 +810,7 @@ class VimInteractionAffinityGroup(VimInteractionBase):
                 task_id,
                 ro_task["target_id"],
                 affinity_group_vim_id,
-                ro_vim_item_update_ok.get("vim_details", ""),
+                ro_vim_item_update_ok.get("vim_message", ""),
             )
         )
 
@@ -852,6 +858,7 @@ class VimInteractionAffinityGroup(VimInteractionBase):
                 "created": created,
                 "created_items": created_items,
                 "vim_details": None,
+                "vim_message": None,
             }
             self.logger.debug(
                 "task={} {} new-affinity-or-anti-affinity-group={} created={}".format(
@@ -868,7 +875,7 @@ class VimInteractionAffinityGroup(VimInteractionBase):
             ro_vim_item_update = {
                 "vim_status": "VIM_ERROR",
                 "created": created,
-                "vim_details": str(e),
+                "vim_message": str(e),
             }
 
             return "FAILED", ro_vim_item_update
@@ -1194,6 +1201,7 @@ class VimInteractionSdnNet(VimInteractionBase):
                 "created_items": created_items,
                 "connected_ports": connected_ports,
                 "vim_details": sdn_info,
+                "vim_message": None,
                 "last_update": last_update,
             }
 
@@ -1208,7 +1216,7 @@ class VimInteractionSdnNet(VimInteractionBase):
             ro_vim_item_update = {
                 "vim_status": "VIM_ERROR",
                 "created": created,
-                "vim_details": str(e),
+                "vim_message": str(e),
             }
 
             return "FAILED", ro_vim_item_update
@@ -1220,7 +1228,7 @@ class VimInteractionSdnNet(VimInteractionBase):
         ro_vim_item_update_ok = {
             "vim_status": "DELETED",
             "created": False,
-            "vim_details": "DELETED",
+            "vim_message": "DELETED",
             "vim_id": None,
         }
 
@@ -1236,7 +1244,7 @@ class VimInteractionSdnNet(VimInteractionBase):
                 isinstance(e, sdnconn.SdnConnectorError)
                 and e.http_code == HTTPStatus.NOT_FOUND.value
             ):
-                ro_vim_item_update_ok["vim_details"] = "already deleted"
+                ro_vim_item_update_ok["vim_message"] = "already deleted"
             else:
                 self.logger.error(
                     "ro_task={} vim={} del-sdn-net={}: {}".format(
@@ -1248,7 +1256,7 @@ class VimInteractionSdnNet(VimInteractionBase):
                 )
                 ro_vim_item_update = {
                     "vim_status": "VIM_ERROR",
-                    "vim_details": "Error while deleting: {}".format(e),
+                    "vim_message": "Error while deleting: {}".format(e),
                 }
 
                 return "FAILED", ro_vim_item_update
@@ -1258,13 +1266,94 @@ class VimInteractionSdnNet(VimInteractionBase):
                 task_id,
                 ro_task["target_id"],
                 sdn_vim_id,
-                ro_vim_item_update_ok.get("vim_details", ""),
+                ro_vim_item_update_ok.get("vim_message", ""),
             )
         )
 
         return "DONE", ro_vim_item_update_ok
 
 
+class VimInteractionMigration(VimInteractionBase):
+    def exec(self, ro_task, task_index, task_depends):
+        task = ro_task["tasks"][task_index]
+        task_id = task["task_id"]
+        db_task_update = {"retries": 0}
+        target_vim = self.my_vims[ro_task["target_id"]]
+        vim_interfaces = []
+        created = False
+        created_items = {}
+        refreshed_vim_info = {}
+
+        try:
+            if task.get("params"):
+                vim_vm_id = task["params"].get("vim_vm_id")
+                migrate_host = task["params"].get("migrate_host")
+                _, migrated_compute_node = target_vim.migrate_instance(
+                    vim_vm_id, migrate_host
+                )
+
+                if migrated_compute_node:
+                    # When VM is migrated, vdu["vim_info"] needs to be updated
+                    vdu_old_vim_info = task["params"]["vdu_vim_info"].get(
+                        ro_task["target_id"]
+                    )
+
+                    # Refresh VM to get new vim_info
+                    vm_to_refresh_list = [vim_vm_id]
+                    vim_dict = target_vim.refresh_vms_status(vm_to_refresh_list)
+                    refreshed_vim_info = vim_dict[vim_vm_id]
+
+                    if refreshed_vim_info.get("interfaces"):
+                        for old_iface in vdu_old_vim_info.get("interfaces"):
+                            iface = next(
+                                (
+                                    iface
+                                    for iface in refreshed_vim_info["interfaces"]
+                                    if old_iface["vim_interface_id"]
+                                    == iface["vim_interface_id"]
+                                ),
+                                None,
+                            )
+                            vim_interfaces.append(iface)
+
+            ro_vim_item_update = {
+                "vim_id": vim_vm_id,
+                "vim_status": "ACTIVE",
+                "created": created,
+                "created_items": created_items,
+                "vim_details": None,
+                "vim_message": None,
+            }
+
+            if refreshed_vim_info and refreshed_vim_info.get("status") not in (
+                "ERROR",
+                "VIM_ERROR",
+            ):
+                ro_vim_item_update["vim_details"] = refreshed_vim_info["vim_info"]
+
+            if vim_interfaces:
+                ro_vim_item_update["interfaces"] = vim_interfaces
+
+            self.logger.debug(
+                "task={} {} vm-migration done".format(task_id, ro_task["target_id"])
+            )
+
+            return "DONE", ro_vim_item_update, db_task_update
+
+        except (vimconn.VimConnException, NsWorkerException) as e:
+            self.logger.error(
+                "task={} vim={} VM Migration:"
+                " {}".format(task_id, ro_task["target_id"], e)
+            )
+            ro_vim_item_update = {
+                "vim_status": "VIM_ERROR",
+                "created": created,
+                "vim_message": str(e),
+            }
+
+            return "FAILED", ro_vim_item_update, db_task_update
+
+
 class NsWorker(threading.Thread):
     REFRESH_BUILD = 5  # 5 seconds
     REFRESH_ACTIVE = 60  # 1 minute
@@ -1312,6 +1401,9 @@ class NsWorker(threading.Thread):
             "affinity-or-anti-affinity-group": VimInteractionAffinityGroup(
                 self.db, self.my_vims, self.db_vims, self.logger
             ),
+            "migrate": VimInteractionMigration(
+                self.db, self.my_vims, self.db_vims, self.logger
+            ),
         }
         self.time_last_task_processed = None
         # lists of tasks to delete because nsrs or vnfrs has been deleted from db
@@ -1729,7 +1821,7 @@ class NsWorker(threading.Thread):
         1642158640.7986135;1642158640.7986135;1642158640.7986135;b134c9494e75:0a
         ;vim:b7ff9e24-8868-4d68-8a57-a59dc11d0327;None;{'created': False,
         'created_items': None, 'vim_id': None, 'vim_name': None, 'vim_status': None,
-        'vim_details': None, 'refresh_at': None};1;SCHEDULED;
+        'vim_details': None, 'vim_message': None, 'refresh_at': None};1;SCHEDULED;
         888f1864-749a-4fc2-bc1a-97c0fffd6a6f;0;888f1864-749a-4fc2-bc1a-97c0fffd6a6f:2;
         CREATE;image;{'filter_dict': {'name': 'ubuntu-os-cloud:image-family:ubuntu-1804-lts'}}
         """
@@ -1849,7 +1941,9 @@ class NsWorker(threading.Thread):
                     needed_delete = False
 
             if needed_delete:
-                self.logger.warning("Deleting ro_task={} task_index={}".format(ro_task, task_index))
+                self.logger.warning(
+                    "Deleting ro_task={} task_index={}".format(ro_task, task_index)
+                )
                 return self.item2class[my_task["item"]].delete(ro_task, task_index)
             else:
                 return "SUPERSEDED", None
@@ -1862,7 +1956,7 @@ class NsWorker(threading.Thread):
                     exc_info=True,
                 )
 
-            return "FAILED", {"vim_status": "VIM_ERROR", "vim_details": str(e)}
+            return "FAILED", {"vim_status": "VIM_ERROR", "vim_message": str(e)}
 
     def _create_task(self, ro_task, task_index, task_depends, db_update):
         """
@@ -1900,7 +1994,7 @@ class NsWorker(threading.Thread):
                     )
 
                 task_status = "FAILED"
-                ro_vim_item_update = {"vim_status": "VIM_ERROR", "vim_details": str(e)}
+                ro_vim_item_update = {"vim_status": "VIM_ERROR", "vim_message": str(e)}
                 # TODO update    ro_vim_item_update
 
             return task_status, ro_vim_item_update
@@ -2054,7 +2148,11 @@ class NsWorker(threading.Thread):
                                 dependency_task = dependency_ro_task["tasks"][
                                     dependency_task_index
                                 ]
-                                self.logger.warning("dependency_ro_task={} dependency_task_index={}".format(dependency_ro_task, dependency_task_index))
+                                self.logger.warning(
+                                    "dependency_ro_task={} dependency_task_index={}".format(
+                                        dependency_ro_task, dependency_task_index
+                                    )
+                                )
 
                                 if dependency_task["status"] == "SCHEDULED":
                                     dependency_not_completed = True
@@ -2075,7 +2173,7 @@ class NsWorker(threading.Thread):
                                         dependency_task["item"],
                                         dependency_task_id,
                                         dependency_ro_task["vim_info"].get(
-                                            "vim_details"
+                                            "vim_message"
                                         ),
                                     )
                                     self.logger.error(
@@ -2091,7 +2189,11 @@ class NsWorker(threading.Thread):
                                 ] = dependency_ro_task["vim_info"]["vim_id"]
 
                             if dependency_not_completed:
-                                self.logger.warning("DEPENDENCY NOT COMPLETED {}".format(dependency_ro_task["vim_info"]["vim_id"]))
+                                self.logger.warning(
+                                    "DEPENDENCY NOT COMPLETED {}".format(
+                                        dependency_ro_task["vim_info"]["vim_id"]
+                                    )
+                                )
                                 # TODO set at vim_info.vim_details that it is waiting
                                 continue
 
@@ -2167,7 +2269,7 @@ class NsWorker(threading.Thread):
                         new_status = "FAILED"
                         db_vim_info_update = {
                             "vim_status": "VIM_ERROR",
-                            "vim_details": str(e),
+                            "vim_message": str(e),
                         }
 
                         if not isinstance(
@@ -2301,7 +2403,14 @@ class NsWorker(threading.Thread):
                 path_vim_status + "." + k: v
                 for k, v in ro_vim_item_update.items()
                 if k
-                in ("vim_id", "vim_details", "vim_name", "vim_status", "interfaces")
+                in (
+                    "vim_id",
+                    "vim_details",
+                    "vim_message",
+                    "vim_name",
+                    "vim_status",
+                    "interfaces",
+                )
             }
 
             if path_vim_status.startswith("vdur."):