A ro_task can contain several 'tasks', each one with a target, where to store the results
"""
+from copy import deepcopy
+from http import HTTPStatus
import logging
+from os import makedirs
+from os import path
import queue
import threading
import time
-import yaml
-from copy import deepcopy
-from http import HTTPStatus
-from os import mkdir
-from importlib_metadata import entry_points
-from shutil import rmtree
+import traceback
+from typing import Dict
from unittest.mock import Mock
-# from osm_common import dbmongo, dbmemory, fslocal, fsmongo, msglocal, msgkafka, version as common_version
+from importlib_metadata import entry_points
from osm_common.dbbase import DbException
-from osm_ro_plugin.vim_dummy import VimDummyConnector
-from osm_ro_plugin.sdn_dummy import SdnDummyConnector
-from osm_ro_plugin import vimconn, sdnconn
from osm_ng_ro.vim_admin import LockRenew
-
+from osm_ro_plugin import sdnconn
+from osm_ro_plugin import vimconn
+from osm_ro_plugin.sdn_dummy import SdnDummyConnector
+from osm_ro_plugin.vim_dummy import VimDummyConnector
+import yaml
__author__ = "Alfonso Tierno"
__date__ = "$28-Sep-2017 12:07:15$"
created = False
created_items = {}
target_vim = self.my_vims[ro_task["target_id"]]
+ mgmtnet = False
+ mgmtnet_defined_in_vim = False
try:
# FIND
# if management, get configuration of VIM
if task["find_params"].get("filter_dict"):
vim_filter = task["find_params"]["filter_dict"]
- # mamagement network
+ # management network
elif task["find_params"].get("mgmt"):
+ mgmtnet = True
if deep_get(
self.db_vims[ro_task["target_id"]],
"config",
"management_network_id",
):
+ mgmtnet_defined_in_vim = True
vim_filter = {
"id": self.db_vims[ro_task["target_id"]]["config"][
"management_network_id"
"config",
"management_network_name",
):
+ mgmtnet_defined_in_vim = True
vim_filter = {
"name": self.db_vims[ro_task["target_id"]]["config"][
"management_network_name"
vim_nets = target_vim.get_network_list(vim_filter)
if not vim_nets and not task.get("params"):
- raise NsWorkerExceptionNotFound(
- "Network not found with this criteria: '{}'".format(
- task.get("find_params")
+ # If there is mgmt-network in the descriptor,
+ # there is no mapping of that network to a VIM network in the descriptor,
+ # also there is no mapping in the "--config" parameter or at VIM creation;
+ # that mgmt-network will be created.
+ if mgmtnet and not mgmtnet_defined_in_vim:
+ net_name = (
+ vim_filter.get("name")
+ if vim_filter.get("name")
+ else vim_filter.get("id")[:16]
+ )
+ vim_net_id, created_items = target_vim.new_network(
+ net_name, None
+ )
+ self.logger.debug(
+ "Created mgmt network vim_net_id: {}".format(vim_net_id)
+ )
+ created = True
+ else:
+ raise NsWorkerExceptionNotFound(
+ "Network not found with this criteria: '{}'".format(
+ task.get("find_params")
+ )
)
- )
elif len(vim_nets) > 1:
raise NsWorkerException(
"More than one network found with this criteria: '{}'".format(
"created": created,
"created_items": created_items,
"vim_details": None,
+ "vim_message": None,
}
self.logger.debug(
"task={} {} new-net={} created={}".format(
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
ro_vim_item_update["vim_name"] = vim_info.get("name")
if vim_info["status"] in ("ERROR", "VIM_ERROR"):
- if ro_task["vim_info"]["vim_details"] != vim_info.get("error_msg"):
- ro_vim_item_update["vim_details"] = vim_info.get("error_msg")
+ if ro_task["vim_info"]["vim_message"] != vim_info.get("error_msg"):
+ ro_vim_item_update["vim_message"] = vim_info.get("error_msg")
elif vim_info["status"] == "DELETED":
ro_vim_item_update["vim_id"] = None
- ro_vim_item_update["vim_details"] = "Deleted externally"
+ ro_vim_item_update["vim_message"] = "Deleted externally"
else:
if ro_task["vim_info"]["vim_details"] != vim_info["vim_info"]:
ro_vim_item_update["vim_details"] = vim_info["vim_info"]
ro_task["target_id"],
vim_id,
ro_vim_item_update.get("vim_status"),
- ro_vim_item_update.get("vim_details")
+ ro_vim_item_update.get("vim_message")
if ro_vim_item_update.get("vim_status") != "ACTIVE"
else "",
)
ro_vim_item_update_ok = {
"vim_status": "DELETED",
"created": False,
- "vim_details": "DELETED",
+ "vim_message": "DELETED",
"vim_id": None,
}
net_vim_id, ro_task["vim_info"]["created_items"]
)
except vimconn.VimConnNotFoundException:
- ro_vim_item_update_ok["vim_details"] = "already deleted"
+ ro_vim_item_update_ok["vim_message"] = "already deleted"
except vimconn.VimConnException as e:
self.logger.error(
"ro_task={} vim={} del-net={}: {}".format(
)
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
- "vim_details": "Error while deleting: {}".format(e),
+ "vim_message": "Error while deleting: {}".format(e),
}
return "FAILED", ro_vim_item_update
task_id,
ro_task["target_id"],
net_vim_id,
- ro_vim_item_update_ok.get("vim_details", ""),
+ ro_vim_item_update_ok.get("vim_message", ""),
)
)
if params_copy["flavor_id"].startswith("TASK-"):
params_copy["flavor_id"] = task_depends[params_copy["flavor_id"]]
+ affinity_group_list = params_copy["affinity_group_list"]
+ for affinity_group in affinity_group_list:
+ # change task_id into affinity_group_id
+ if "affinity_group_id" in affinity_group and affinity_group[
+ "affinity_group_id"
+ ].startswith("TASK-"):
+ affinity_group_id = task_depends[
+ affinity_group["affinity_group_id"]
+ ]
+
+ if not affinity_group_id:
+ raise NsWorkerException(
+ "found for {}".format(affinity_group["affinity_group_id"])
+ )
+
+ affinity_group["affinity_group_id"] = affinity_group_id
+
vim_vm_id, created_items = target_vim.new_vminstance(**params_copy)
interfaces = [iface["vim_id"] for iface in params_copy["net_list"]]
+ # add to created items previous_created_volumes (healing)
+ if task.get("previous_created_volumes"):
+ for k, v in task["previous_created_volumes"].items():
+ created_items[k] = v
+
ro_vim_item_update = {
"vim_id": vim_vm_id,
"vim_status": "BUILD",
"created": created,
"created_items": created_items,
"vim_details": None,
+ "vim_message": None,
"interfaces_vim_ids": interfaces,
"interfaces": [],
+ "interfaces_backup": [],
}
self.logger.debug(
"task={} {} new-vm={} created={}".format(
return "BUILD", ro_vim_item_update
except (vimconn.VimConnException, NsWorkerException) as e:
+ self.logger.debug(traceback.format_exc())
self.logger.error(
"task={} {} new-vm: {}".format(task_id, ro_task["target_id"], e)
)
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
ro_vim_item_update_ok = {
"vim_status": "DELETED",
"created": False,
- "vim_details": "DELETED",
+ "vim_message": "DELETED",
"vim_id": None,
}
try:
+ self.logger.debug(
+ "delete_vminstance: vm_vim_id={} created_items={}".format(
+ vm_vim_id, ro_task["vim_info"]["created_items"]
+ )
+ )
if vm_vim_id or ro_task["vim_info"]["created_items"]:
target_vim = self.my_vims[ro_task["target_id"]]
target_vim.delete_vminstance(
- vm_vim_id, ro_task["vim_info"]["created_items"]
+ vm_vim_id,
+ ro_task["vim_info"]["created_items"],
+ ro_task["vim_info"].get("volumes_to_hold", []),
)
except vimconn.VimConnNotFoundException:
- ro_vim_item_update_ok["vim_details"] = "already deleted"
+ ro_vim_item_update_ok["vim_message"] = "already deleted"
except vimconn.VimConnException as e:
self.logger.error(
"ro_task={} vim={} del-vm={}: {}".format(
)
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
- "vim_details": "Error while deleting: {}".format(e),
+ "vim_message": "Error while deleting: {}".format(e),
}
return "FAILED", ro_vim_item_update
task_id,
ro_task["target_id"],
vm_vim_id,
- ro_vim_item_update_ok.get("vim_details", ""),
+ ro_vim_item_update_ok.get("vim_message", ""),
)
)
vim_info_info = yaml.safe_load(vim_info["vim_info"])
if vim_info_info.get("name"):
vim_info["name"] = vim_info_info["name"]
- except Exception:
- pass
+ except Exception as vim_info_error:
+ self.logger.exception(
+ f"{vim_info_error} occured while getting the vim_info from yaml"
+ )
except vimconn.VimConnException as e:
# Mark all tasks at VIM_ERROR status
self.logger.error(
ro_vim_item_update["vim_name"] = vim_info.get("name")
if vim_info["status"] in ("ERROR", "VIM_ERROR"):
- if ro_task["vim_info"]["vim_details"] != vim_info.get("error_msg"):
- ro_vim_item_update["vim_details"] = vim_info.get("error_msg")
+ if ro_task["vim_info"]["vim_message"] != vim_info.get("error_msg"):
+ ro_vim_item_update["vim_message"] = vim_info.get("error_msg")
elif vim_info["status"] == "DELETED":
ro_vim_item_update["vim_id"] = None
- ro_vim_item_update["vim_details"] = "Deleted externally"
+ ro_vim_item_update["vim_message"] = "Deleted externally"
else:
if ro_task["vim_info"]["vim_details"] != vim_info["vim_info"]:
ro_vim_item_update["vim_details"] = vim_info["vim_info"]
ro_task["target_id"],
vim_id,
ro_vim_item_update.get("vim_status"),
- ro_vim_item_update.get("vim_details")
+ ro_vim_item_update.get("vim_message")
if ro_vim_item_update.get("vim_status") != "ACTIVE"
else "",
)
except (vimconn.VimConnException, NsWorkerException) as e:
retries += 1
+ self.logger.debug(traceback.format_exc())
if retries < self.max_retries_inject_ssh_key:
return (
"BUILD",
self.logger.error(
"task={} {} inject-ssh-key: {}".format(task_id, ro_task["target_id"], e)
)
- ro_vim_item_update = {"vim_details": str(e)}
+ ro_vim_item_update = {"vim_message": str(e)}
return "FAILED", ro_vim_item_update, db_task_update
"created": created,
"created_items": created_items,
"vim_details": None,
+ "vim_message": None,
}
self.logger.debug(
"task={} {} new-image={} created={}".format(
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
ro_vim_item_update_ok = {
"vim_status": "DELETED",
"created": False,
- "vim_details": "DELETED",
+ "vim_message": "DELETED",
"vim_id": None,
}
target_vim = self.my_vims[ro_task["target_id"]]
target_vim.delete_flavor(flavor_vim_id)
except vimconn.VimConnNotFoundException:
- ro_vim_item_update_ok["vim_details"] = "already deleted"
+ ro_vim_item_update_ok["vim_message"] = "already deleted"
except vimconn.VimConnException as e:
self.logger.error(
"ro_task={} vim={} del-flavor={}: {}".format(
)
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
- "vim_details": "Error while deleting: {}".format(e),
+ "vim_message": "Error while deleting: {}".format(e),
}
return "FAILED", ro_vim_item_update
task_id,
ro_task["target_id"],
flavor_vim_id,
- ro_vim_item_update_ok.get("vim_details", ""),
+ ro_vim_item_update_ok.get("vim_message", ""),
)
)
try:
flavor_data = task["find_params"]["flavor_data"]
vim_flavor_id = target_vim.get_flavor_id_from_data(flavor_data)
- except vimconn.VimConnNotFoundException:
- pass
+ except vimconn.VimConnNotFoundException as flavor_not_found_msg:
+ self.logger.warning(
+ f"VimConnNotFoundException occured: {flavor_not_found_msg}"
+ )
if not vim_flavor_id and task.get("params"):
# CREATE
"created": created,
"created_items": created_items,
"vim_details": None,
+ "vim_message": None,
}
self.logger.debug(
"task={} {} new-flavor={} created={}".format(
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
+ }
+
+ return "FAILED", ro_vim_item_update
+
+
+class VimInteractionAffinityGroup(VimInteractionBase):
+ def delete(self, ro_task, task_index):
+ task = ro_task["tasks"][task_index]
+ task_id = task["task_id"]
+ affinity_group_vim_id = ro_task["vim_info"]["vim_id"]
+ ro_vim_item_update_ok = {
+ "vim_status": "DELETED",
+ "created": False,
+ "vim_message": "DELETED",
+ "vim_id": None,
+ }
+
+ try:
+ if affinity_group_vim_id:
+ target_vim = self.my_vims[ro_task["target_id"]]
+ target_vim.delete_affinity_group(affinity_group_vim_id)
+ except vimconn.VimConnNotFoundException:
+ ro_vim_item_update_ok["vim_message"] = "already deleted"
+ except vimconn.VimConnException as e:
+ self.logger.error(
+ "ro_task={} vim={} del-affinity-or-anti-affinity-group={}: {}".format(
+ ro_task["_id"], ro_task["target_id"], affinity_group_vim_id, e
+ )
+ )
+ ro_vim_item_update = {
+ "vim_status": "VIM_ERROR",
+ "vim_message": "Error while deleting: {}".format(e),
+ }
+
+ return "FAILED", ro_vim_item_update
+
+ self.logger.debug(
+ "task={} {} del-affinity-or-anti-affinity-group={} {}".format(
+ task_id,
+ ro_task["target_id"],
+ affinity_group_vim_id,
+ ro_vim_item_update_ok.get("vim_message", ""),
+ )
+ )
+
+ return "DONE", ro_vim_item_update_ok
+
+ def new(self, ro_task, task_index, task_depends):
+ task = ro_task["tasks"][task_index]
+ task_id = task["task_id"]
+ created = False
+ created_items = {}
+ target_vim = self.my_vims[ro_task["target_id"]]
+
+ try:
+ affinity_group_vim_id = None
+ affinity_group_data = None
+
+ if task.get("params"):
+ affinity_group_data = task["params"].get("affinity_group_data")
+
+ if affinity_group_data and affinity_group_data.get("vim-affinity-group-id"):
+ try:
+ param_affinity_group_id = task["params"]["affinity_group_data"].get(
+ "vim-affinity-group-id"
+ )
+ affinity_group_vim_id = target_vim.get_affinity_group(
+ param_affinity_group_id
+ ).get("id")
+ except vimconn.VimConnNotFoundException:
+ self.logger.error(
+ "task={} {} new-affinity-or-anti-affinity-group. Provided VIM Affinity Group ID {}"
+ "could not be found at VIM. Creating a new one.".format(
+ task_id, ro_task["target_id"], param_affinity_group_id
+ )
+ )
+
+ if not affinity_group_vim_id and affinity_group_data:
+ affinity_group_vim_id = target_vim.new_affinity_group(
+ affinity_group_data
+ )
+ created = True
+
+ ro_vim_item_update = {
+ "vim_id": affinity_group_vim_id,
+ "vim_status": "DONE",
+ "created": created,
+ "created_items": created_items,
+ "vim_details": None,
+ "vim_message": None,
+ }
+ self.logger.debug(
+ "task={} {} new-affinity-or-anti-affinity-group={} created={}".format(
+ task_id, ro_task["target_id"], affinity_group_vim_id, created
+ )
+ )
+
+ return "DONE", ro_vim_item_update
+ except (vimconn.VimConnException, NsWorkerException) as e:
+ self.logger.error(
+ "task={} vim={} new-affinity-or-anti-affinity-group:"
+ " {}".format(task_id, ro_task["target_id"], e)
+ )
+ ro_vim_item_update = {
+ "vim_status": "VIM_ERROR",
+ "created": created,
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
+class VimInteractionUpdateVdu(VimInteractionBase):
+ def exec(self, ro_task, task_index, task_depends):
+ task = ro_task["tasks"][task_index]
+ task_id = task["task_id"]
+ db_task_update = {"retries": 0}
+ created = False
+ created_items = {}
+ target_vim = self.my_vims[ro_task["target_id"]]
+
+ try:
+ if task.get("params"):
+ vim_vm_id = task["params"].get("vim_vm_id")
+ action = task["params"].get("action")
+ context = {action: action}
+ target_vim.action_vminstance(vim_vm_id, context)
+ # created = True
+ ro_vim_item_update = {
+ "vim_id": vim_vm_id,
+ "vim_status": "DONE",
+ "created": created,
+ "created_items": created_items,
+ "vim_details": None,
+ "vim_message": None,
+ }
+ self.logger.debug(
+ "task={} {} vm-migration done".format(task_id, ro_task["target_id"])
+ )
+ return "DONE", ro_vim_item_update, db_task_update
+ except (vimconn.VimConnException, NsWorkerException) as e:
+ self.logger.error(
+ "task={} vim={} VM Migration:"
+ " {}".format(task_id, ro_task["target_id"], e)
+ )
+ ro_vim_item_update = {
+ "vim_status": "VIM_ERROR",
+ "created": created,
+ "vim_message": str(e),
+ }
+
+ return "FAILED", ro_vim_item_update, db_task_update
+
+
class VimInteractionSdnNet(VimInteractionBase):
@staticmethod
def _match_pci(port_pci, mapping):
return self.new(ro_task, task_create_index, None)
def new(self, ro_task, task_index, task_depends):
-
task = ro_task["tasks"][task_index]
task_id = task["task_id"]
target_vim = self.my_vims[ro_task["target_id"]]
try:
# CREATE
params = task["params"]
- vlds_to_connect = params["vlds"]
- associated_vim = params["target_vim"]
+ vlds_to_connect = params.get("vlds", [])
+ associated_vim = params.get("target_vim")
# external additional ports
additional_ports = params.get("sdn-ports") or ()
- _, _, vim_account_id = associated_vim.partition(":")
+ _, _, vim_account_id = (
+ (None, None, None)
+ if associated_vim is None
+ else associated_vim.partition(":")
+ )
if associated_vim:
# get associated VIM
"created_items": created_items,
"connected_ports": connected_ports,
"vim_details": sdn_info,
+ "vim_message": None,
"last_update": last_update,
}
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
"created": created,
- "vim_details": str(e),
+ "vim_message": str(e),
}
return "FAILED", ro_vim_item_update
ro_vim_item_update_ok = {
"vim_status": "DELETED",
"created": False,
- "vim_details": "DELETED",
+ "vim_message": "DELETED",
"vim_id": None,
}
isinstance(e, sdnconn.SdnConnectorError)
and e.http_code == HTTPStatus.NOT_FOUND.value
):
- ro_vim_item_update_ok["vim_details"] = "already deleted"
+ ro_vim_item_update_ok["vim_message"] = "already deleted"
else:
self.logger.error(
"ro_task={} vim={} del-sdn-net={}: {}".format(
)
ro_vim_item_update = {
"vim_status": "VIM_ERROR",
- "vim_details": "Error while deleting: {}".format(e),
+ "vim_message": "Error while deleting: {}".format(e),
}
return "FAILED", ro_vim_item_update
task_id,
ro_task["target_id"],
sdn_vim_id,
- ro_vim_item_update_ok.get("vim_details", ""),
+ ro_vim_item_update_ok.get("vim_message", ""),
)
)
return "DONE", ro_vim_item_update_ok
-class NsWorker(threading.Thread):
- REFRESH_BUILD = 5 # 5 seconds
- REFRESH_ACTIVE = 60 # 1 minute
- REFRESH_ERROR = 600
- REFRESH_IMAGE = 3600 * 10
- REFRESH_DELETE = 3600 * 10
- QUEUE_SIZE = 100
- terminate = False
+class VimInteractionMigration(VimInteractionBase):
+ def exec(self, ro_task, task_index, task_depends):
+ task = ro_task["tasks"][task_index]
+ task_id = task["task_id"]
+ db_task_update = {"retries": 0}
+ target_vim = self.my_vims[ro_task["target_id"]]
+ vim_interfaces = []
+ created = False
+ created_items = {}
+ refreshed_vim_info = {}
+
+ try:
+ if task.get("params"):
+ vim_vm_id = task["params"].get("vim_vm_id")
+ migrate_host = task["params"].get("migrate_host")
+ _, migrated_compute_node = target_vim.migrate_instance(
+ vim_vm_id, migrate_host
+ )
+
+ if migrated_compute_node:
+ # When VM is migrated, vdu["vim_info"] needs to be updated
+ vdu_old_vim_info = task["params"]["vdu_vim_info"].get(
+ ro_task["target_id"]
+ )
+
+ # Refresh VM to get new vim_info
+ vm_to_refresh_list = [vim_vm_id]
+ vim_dict = target_vim.refresh_vms_status(vm_to_refresh_list)
+ refreshed_vim_info = vim_dict[vim_vm_id]
+
+ if refreshed_vim_info.get("interfaces"):
+ for old_iface in vdu_old_vim_info.get("interfaces"):
+ iface = next(
+ (
+ iface
+ for iface in refreshed_vim_info["interfaces"]
+ if old_iface["vim_interface_id"]
+ == iface["vim_interface_id"]
+ ),
+ None,
+ )
+ vim_interfaces.append(iface)
+
+ ro_vim_item_update = {
+ "vim_id": vim_vm_id,
+ "vim_status": "ACTIVE",
+ "created": created,
+ "created_items": created_items,
+ "vim_details": None,
+ "vim_message": None,
+ }
+
+ if refreshed_vim_info and refreshed_vim_info.get("status") not in (
+ "ERROR",
+ "VIM_ERROR",
+ ):
+ ro_vim_item_update["vim_details"] = refreshed_vim_info["vim_info"]
+
+ if vim_interfaces:
+ ro_vim_item_update["interfaces"] = vim_interfaces
+
+ self.logger.debug(
+ "task={} {} vm-migration done".format(task_id, ro_task["target_id"])
+ )
+
+ return "DONE", ro_vim_item_update, db_task_update
+
+ except (vimconn.VimConnException, NsWorkerException) as e:
+ self.logger.error(
+ "task={} vim={} VM Migration:"
+ " {}".format(task_id, ro_task["target_id"], e)
+ )
+ ro_vim_item_update = {
+ "vim_status": "VIM_ERROR",
+ "created": created,
+ "vim_message": str(e),
+ }
+
+ return "FAILED", ro_vim_item_update, db_task_update
+
+
+class VimInteractionResize(VimInteractionBase):
+ def exec(self, ro_task, task_index, task_depends):
+ task = ro_task["tasks"][task_index]
+ task_id = task["task_id"]
+ db_task_update = {"retries": 0}
+ created = False
+ target_flavor_uuid = None
+ created_items = {}
+ refreshed_vim_info = {}
+ target_vim = self.my_vims[ro_task["target_id"]]
+
+ try:
+ if task.get("params"):
+ vim_vm_id = task["params"].get("vim_vm_id")
+ flavor_dict = task["params"].get("flavor_dict")
+ self.logger.info("flavor_dict %s", flavor_dict)
+
+ try:
+ target_flavor_uuid = target_vim.get_flavor_id_from_data(flavor_dict)
+ except Exception as e:
+ self.logger.info("Cannot find any flavor matching %s.", str(e))
+ try:
+ target_flavor_uuid = target_vim.new_flavor(flavor_dict)
+ except Exception as e:
+ self.logger.error("Error creating flavor at VIM %s.", str(e))
+
+ if target_flavor_uuid is not None:
+ resized_status = target_vim.resize_instance(
+ vim_vm_id, target_flavor_uuid
+ )
+
+ if resized_status:
+ # Refresh VM to get new vim_info
+ vm_to_refresh_list = [vim_vm_id]
+ vim_dict = target_vim.refresh_vms_status(vm_to_refresh_list)
+ refreshed_vim_info = vim_dict[vim_vm_id]
+
+ ro_vim_item_update = {
+ "vim_id": vim_vm_id,
+ "vim_status": "DONE",
+ "created": created,
+ "created_items": created_items,
+ "vim_details": None,
+ "vim_message": None,
+ }
+
+ if refreshed_vim_info and refreshed_vim_info.get("status") not in (
+ "ERROR",
+ "VIM_ERROR",
+ ):
+ ro_vim_item_update["vim_details"] = refreshed_vim_info["vim_info"]
+
+ self.logger.debug(
+ "task={} {} resize done".format(task_id, ro_task["target_id"])
+ )
+ return "DONE", ro_vim_item_update, db_task_update
+ except (vimconn.VimConnException, NsWorkerException) as e:
+ self.logger.error(
+ "task={} vim={} Resize:" " {}".format(task_id, ro_task["target_id"], e)
+ )
+ ro_vim_item_update = {
+ "vim_status": "VIM_ERROR",
+ "created": created,
+ "vim_message": str(e),
+ }
+
+ return "FAILED", ro_vim_item_update, db_task_update
+
+
+class ConfigValidate:
+ def __init__(self, config: Dict):
+ self.conf = config
+
+ @property
+ def active(self):
+ # default 1 min, allowed >= 60 or -1, -1 disables periodic checks
+ if (
+ self.conf["period"]["refresh_active"] >= 60
+ or self.conf["period"]["refresh_active"] == -1
+ ):
+ return self.conf["period"]["refresh_active"]
+ return 60
+
+ @property
+ def build(self):
+ return self.conf["period"]["refresh_build"]
+
+ @property
+ def image(self):
+ return self.conf["period"]["refresh_image"]
+
+ @property
+ def error(self):
+ return self.conf["period"]["refresh_error"]
+
+ @property
+ def queue_size(self):
+ return self.conf["period"]["queue_size"]
+
+
+class NsWorker(threading.Thread):
def __init__(self, worker_index, config, plugins, db):
"""
-
:param worker_index: thread index
:param config: general configuration of RO, among others the process_id with the docker id where it runs
:param plugins: global shared dict with the loaded plugins
self.plugin_name = "unknown"
self.logger = logging.getLogger("ro.worker{}".format(worker_index))
self.worker_index = worker_index
- self.task_queue = queue.Queue(self.QUEUE_SIZE)
+ # refresh periods for created items
+ self.refresh_config = ConfigValidate(config)
+ self.task_queue = queue.Queue(self.refresh_config.queue_size)
# targetvim: vimplugin class
self.my_vims = {}
# targetvim: vim information from database
"sdn_net": VimInteractionSdnNet(
self.db, self.my_vims, self.db_vims, self.logger
),
+ "update": VimInteractionUpdateVdu(
+ self.db, self.my_vims, self.db_vims, self.logger
+ ),
+ "affinity-or-anti-affinity-group": VimInteractionAffinityGroup(
+ self.db, self.my_vims, self.db_vims, self.logger
+ ),
+ "migrate": VimInteractionMigration(
+ self.db, self.my_vims, self.db_vims, self.logger
+ ),
+ "verticalscale": VimInteractionResize(
+ self.db, self.my_vims, self.db_vims, self.logger
+ ),
}
self.time_last_task_processed = None
# lists of tasks to delete because nsrs or vnfrs has been deleted from db
self.task_lock.release()
return False
- def _process_vim_config(self, target_id, db_vim):
+ def _process_vim_config(self, target_id: str, db_vim: dict) -> None:
"""
Process vim config, creating vim configuration files as ca_cert
:param target_id: vim/sdn/wim + id
return
file_name = ""
+ work_dir = "/app/osm_ro/certs"
try:
if db_vim["config"].get("ca_cert_content"):
- file_name = "{}:{}".format(target_id, self.worker_index)
+ file_name = f"{work_dir}/{target_id}:{self.worker_index}"
- try:
- mkdir(file_name)
- except FileExistsError:
- pass
+ if not path.isdir(file_name):
+ makedirs(file_name)
file_name = file_name + "/ca_cert"
self.vim_targets.remove(target_id)
self.logger.info("Unloaded {}".format(target_id))
- rmtree("{}:{}".format(target_id, self.worker_index))
- except FileNotFoundError:
- pass # this is raised by rmtree if folder does not exist
except Exception as e:
self.logger.error("Cannot unload {}: {}".format(target_id, e))
)
plugin_name = ""
vim = None
+ step = "Getting {}={} from db".format(target, _id)
try:
- step = "Getting {}={} from db".format(target, _id)
# TODO process for wim, sdnc, ...
vim = self.db.get_one(target_database, {"_id": _id})
persistent_info={},
)
else: # sdn
- plugin_name = "rosdn_" + vim["type"]
+ plugin_name = "rosdn_" + (vim.get("type") or vim.get("wim_type"))
step = "Loading plugin '{}'".format(plugin_name)
vim_module_conn = self._load_plugin(plugin_name, "sdn")
step = "Loading {}'".format(target_id)
wim = deepcopy(vim)
wim_config = wim.pop("config", {}) or {}
wim["uuid"] = wim["_id"]
- wim["wim_url"] = wim["url"]
+ if "url" in wim and "wim_url" not in wim:
+ wim["wim_url"] = wim["url"]
+ elif "url" not in wim and "wim_url" in wim:
+ wim["url"] = wim["wim_url"]
if wim.get("dpid"):
wim_config["dpid"] = wim.pop("dpid")
try:
while True:
+ """
+ # Log RO tasks only when loglevel is DEBUG
+ if self.logger.getEffectiveLevel() == logging.DEBUG:
+ self._log_ro_task(
+ None,
+ None,
+ None,
+ "TASK_WF",
+ "task_locked_time="
+ + str(self.task_locked_time)
+ + " "
+ + "time_last_task_processed="
+ + str(self.time_last_task_processed)
+ + " "
+ + "now="
+ + str(now),
+ )
+ """
locked = self.db.set_one(
"ro_tasks",
q_filter={
"tasks.status": ["SCHEDULED", "BUILD", "DONE", "FAILED"],
"locked_at.lt": now - self.task_locked_time,
"to_check_at.lt": self.time_last_task_processed,
+ "to_check_at.gt": -1,
},
update_dict={"locked_by": self.my_id, "locked_at": now},
fail_on_empty=False,
"created_items", False
)
+ self.logger.debug("Needed delete: {}".format(needed_delete))
if my_task["status"] == "FAILED":
return None, None # TODO need to be retry??
needed_delete = False
if needed_delete:
+ self.logger.debug(
+ "Deleting ro_task={} task_index={}".format(ro_task, task_index)
+ )
return self.item2class[my_task["item"]].delete(ro_task, task_index)
else:
return "SUPERSEDED", None
exc_info=True,
)
- return "FAILED", {"vim_status": "VIM_ERROR", "vim_details": str(e)}
+ return "FAILED", {"vim_status": "VIM_ERROR", "vim_message": str(e)}
def _create_task(self, ro_task, task_index, task_depends, db_update):
"""
)
task_status = "FAILED"
- ro_vim_item_update = {"vim_status": "VIM_ERROR", "vim_details": str(e)}
+ ro_vim_item_update = {"vim_status": "VIM_ERROR", "vim_message": str(e)}
# TODO update ro_vim_item_update
return task_status, ro_vim_item_update
fail_on_empty=False,
)
+ self.logger.debug("ro_task_dependency={}".format(ro_task_dependency))
if ro_task_dependency:
- for task_index, task in ro_task_dependency["tasks"]:
+ for task_index, task in enumerate(ro_task_dependency["tasks"]):
if task["task_id"] == task_id:
return ro_task_dependency, task_index
raise NsWorkerException("Cannot get depending task {}".format(task_id))
+ def update_vm_refresh(self, ro_task):
+ """Enables the VM status updates if self.refresh_config.active parameter
+ is not -1 and then updates the DB accordingly
+
+ """
+ try:
+ self.logger.debug("Checking if VM status update config")
+ next_refresh = time.time()
+ next_refresh = self._get_next_refresh(ro_task, next_refresh)
+
+ if next_refresh != -1:
+ db_ro_task_update = {}
+ now = time.time()
+ next_check_at = now + (24 * 60 * 60)
+ next_check_at = min(next_check_at, next_refresh)
+ db_ro_task_update["vim_info.refresh_at"] = next_refresh
+ db_ro_task_update["to_check_at"] = next_check_at
+
+ self.logger.debug(
+ "Finding tasks which to be updated to enable VM status updates"
+ )
+ refresh_tasks = self.db.get_list(
+ "ro_tasks",
+ q_filter={
+ "tasks.status": "DONE",
+ "to_check_at.lt": 0,
+ },
+ )
+ self.logger.debug("Updating tasks to change the to_check_at status")
+ for task in refresh_tasks:
+ q_filter = {
+ "_id": task["_id"],
+ }
+ self.db.set_one(
+ "ro_tasks",
+ q_filter=q_filter,
+ update_dict=db_ro_task_update,
+ fail_on_empty=True,
+ )
+
+ except Exception as e:
+ self.logger.error(f"Error updating tasks to enable VM status updates: {e}")
+
+ def _get_next_refresh(self, ro_task: dict, next_refresh: float):
+ """Decide the next_refresh according to vim type and refresh config period.
+ Args:
+ ro_task (dict): ro_task details
+ next_refresh (float): next refresh time as epoch format
+
+ Returns:
+ next_refresh (float) -1 if vm updates are disabled or vim type is openstack.
+ """
+ target_vim = ro_task["target_id"]
+ vim_type = self.db_vims[target_vim]["vim_type"]
+ if self.refresh_config.active == -1 or vim_type == "openstack":
+ next_refresh = -1
+ else:
+ next_refresh += self.refresh_config.active
+ return next_refresh
+
def _process_pending_tasks(self, ro_task):
ro_task_id = ro_task["_id"]
now = time.time()
next_refresh = time.time()
if task["item"] in ("image", "flavor"):
- next_refresh += self.REFRESH_IMAGE
+ next_refresh += self.refresh_config.image
elif new_status == "BUILD":
- next_refresh += self.REFRESH_BUILD
+ next_refresh += self.refresh_config.build
elif new_status == "DONE":
- next_refresh += self.REFRESH_ACTIVE
+ next_refresh = self._get_next_refresh(ro_task, next_refresh)
else:
- next_refresh += self.REFRESH_ERROR
+ next_refresh += self.refresh_config.error
next_check_at = min(next_check_at, next_refresh)
db_ro_task_update["vim_info.refresh_at"] = next_refresh
ro_task["vim_info"]["refresh_at"] = next_refresh
try:
+ """
+ # Log RO tasks only when loglevel is DEBUG
+ if self.logger.getEffectiveLevel() == logging.DEBUG:
+ self._log_ro_task(ro_task, None, None, "TASK_WF", "GET_TASK")
+ """
+ # Check if vim status refresh is enabled again
+ self.update_vm_refresh(ro_task)
# 0: get task_status_create
lock_object = None
task_status_create = None
dependency_task = dependency_ro_task["tasks"][
dependency_task_index
]
+ self.logger.debug(
+ "dependency_ro_task={} dependency_task_index={}".format(
+ dependency_ro_task, dependency_task_index
+ )
+ )
if dependency_task["status"] == "SCHEDULED":
dependency_not_completed = True
dependency_task["item"],
dependency_task_id,
dependency_ro_task["vim_info"].get(
- "vim_details"
+ "vim_message"
),
)
self.logger.error(
] = dependency_ro_task["vim_info"]["vim_id"]
if dependency_not_completed:
+ self.logger.warning(
+ "DEPENDENCY NOT COMPLETED {}".format(
+ dependency_ro_task["vim_info"]["vim_id"]
+ )
+ )
# TODO set at vim_info.vim_details that it is waiting
continue
)
if task["action"] == "DELETE":
- (new_status, db_vim_info_update,) = self._delete_task(
+ (
+ new_status,
+ db_vim_info_update,
+ ) = self._delete_task(
ro_task, task_index, task_depends, db_ro_task_update
)
new_status = (
# self._create_task(ro_task, task_index, task_depends, db_ro_task_update)
_update_refresh(new_status)
else:
- if (
- ro_task["vim_info"]["refresh_at"]
- and now > ro_task["vim_info"]["refresh_at"]
- ):
- new_status, db_vim_info_update = self.item2class[
+ refresh_at = ro_task["vim_info"]["refresh_at"]
+ if refresh_at and refresh_at != -1 and now > refresh_at:
+ (
+ new_status,
+ db_vim_info_update,
+ ) = self.item2class[
task["item"]
].refresh(ro_task)
_update_refresh(new_status)
+ else:
+ # The refresh is updated to avoid set the value of "refresh_at" to
+ # default value (next_check_at = now + (24 * 60 * 60)) when status is BUILD,
+ # because it can happen that in this case the task is never processed
+ _update_refresh(task["status"])
except Exception as e:
new_status = "FAILED"
db_vim_info_update = {
"vim_status": "VIM_ERROR",
- "vim_details": str(e),
+ "vim_message": str(e),
}
if not isinstance(
db_ro_task_update["modified_at"] = now
db_ro_task_update["to_check_at"] = next_check_at
+ """
+ # Log RO tasks only when loglevel is DEBUG
+ if self.logger.getEffectiveLevel() == logging.DEBUG:
+ db_ro_task_update_log = db_ro_task_update.copy()
+ db_ro_task_update_log["_id"] = q_filter["_id"]
+ self._log_ro_task(None, db_ro_task_update_log, None, "TASK_WF", "SET_TASK")
+ """
+
if not self.db.set_one(
"ro_tasks",
update_dict=db_ro_task_update,
):
del db_ro_task_update["to_check_at"]
del q_filter["to_check_at"]
+ """
+ # Log RO tasks only when loglevel is DEBUG
+ if self.logger.getEffectiveLevel() == logging.DEBUG:
+ self._log_ro_task(
+ None,
+ db_ro_task_update_log,
+ None,
+ "TASK_WF",
+ "SET_TASK " + str(q_filter),
+ )
+ """
self.db.set_one(
"ro_tasks",
q_filter=q_filter,
path_vim_status + "." + k: v
for k, v in ro_vim_item_update.items()
if k
- in ("vim_id", "vim_details", "vim_name", "vim_status", "interfaces")
+ in (
+ "vim_id",
+ "vim_details",
+ "vim_message",
+ "vim_name",
+ "vim_status",
+ "interfaces",
+ "interfaces_backup",
+ )
}
if path_vim_status.startswith("vdur."):
).split(";")[0]
self.db.set_one(table, q_filter={"_id": _id}, update_dict=update_dict)
+
+ # If interfaces exists, it backups VDU interfaces in the DB for healing operations
+ if ro_vim_item_update.get("interfaces"):
+ search_key = path_vim_status + ".interfaces"
+ if update_dict.get(search_key):
+ interfaces_backup_update = {
+ path_vim_status + ".interfaces_backup": update_dict[search_key]
+ }
+
+ self.db.set_one(
+ table,
+ q_filter={"_id": _id},
+ update_dict=interfaces_backup_update,
+ )
+
else:
update_dict = {path_item + ".status": "DELETED"}
self.db.set_one(
if self.tasks_to_delete:
self._process_delete_db_tasks()
busy = False
+ """
+ # Log RO tasks only when loglevel is DEBUG
+ if self.logger.getEffectiveLevel() == logging.DEBUG:
+ _ = self._get_db_all_tasks()
+ """
ro_task = self._get_db_task()
if ro_task:
+ self.logger.debug("Task to process: {}".format(ro_task))
+ time.sleep(1)
self._process_pending_tasks(ro_task)
busy = True
if not busy: