osm_lcm/netslice.py

   1 # -*- coding: utf-8 -*-
   2
   3 import asyncio
   4 import logging
   5 import logging.handlers
   6 import traceback
   7 import ns
   8 from lcm_utils import LcmException, LcmBase
   9 from osm_common.dbbase import DbException
  10 from time import time
  11
  12 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
  13
  14
  15 class NetsliceLcm(LcmBase):
  16
  17     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  18         """
  19         Init, Connect to database, filesystem storage, and messaging
  20         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  21         :return: None
  22         """
  23         # logging
  24         self.logger = logging.getLogger('lcm.netslice')
  25         self.loop = loop
  26         self.lcm_tasks = lcm_tasks
  27         self.ns = ns.NsLcm(db, msg, fs, lcm_tasks, ro_config, vca_config, loop)
  28
  29         super().__init__(db, msg, fs, self.logger)
  30
  31     # TODO: check logging_text within the self.logger.info/debug
  32     async def instantiate(self, nsir_id, nsilcmop_id):
  33         logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
  34         self.logger.debug(logging_text + "Enter")
  35         # get all needed from database
  36         exc = None
  37         db_nsir = None
  38         db_nsilcmop = None
  39         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
  40         db_nsilcmop_update = {}
  41         nsilcmop_operation_state = None
  42
  43         try:
  44             step = "Getting nsir={} from db".format(nsir_id)
  45             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
  46             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
  47             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
  48
  49             # look if previous tasks is in process
  50             task_name, task_dependency = self.lcm_tasks.lookfor_related("nsi", nsir_id, nsilcmop_id)
  51             if task_dependency:
  52                 step = db_nsilcmop_update["detailed-status"] = \
  53                     "Waiting for related tasks to be completed: {}".format(task_name)
  54                 self.logger.debug(logging_text + step)
  55                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
  56                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
  57                 if pending:
  58                     raise LcmException("Timeout waiting related tasks to be completed")
  59
  60             # Empty list to keep track of network service records status in the netslice
  61             nsir_admin = db_nsir["_admin"]
  62
  63             nsir_admin["nsrs-detailed-list"] = []
  64
  65             # Slice status Creating
  66             db_nsir_update["detailed-status"] = "creating"
  67             db_nsir_update["operational-status"] = "init"
  68             self.update_db_2("nsis", nsir_id, db_nsir_update)
  69
  70             # Iterate over the network services operation ids to instantiate NSs
  71             # TODO: (future improvement) look another way check the tasks instead of keep asking
  72             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
  73             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
  74             # ns_tasks = []
  75             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
  76             for nslcmop_id in nslcmop_ids:
  77                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
  78                 nsr_id = nslcmop.get("nsInstanceId")
  79                 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop)
  80                 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
  81                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
  82
  83             # Wait until Network Slice is ready
  84             step = nsir_status_detailed = " Waiting nsi ready. nsi_id={}".format(nsir_id)
  85             nsrs_detailed_list_old = None
  86             self.logger.debug(logging_text + step)
  87
  88             # TODO: substitute while for await (all task to be done or not)
  89             deployment_timeout = 2 * 3600   # Two hours
  90             while deployment_timeout > 0:
  91                 # Check ns instantiation status
  92                 nsi_ready = True
  93                 nsrs_detailed_list = []
  94                 for nslcmop_item in nslcmop_ids:
  95                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
  96                     status = nslcmop.get("operationState")
  97                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
  98                     nsrs_detailed_list.append({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
  99                                                "detailed-status":
 100                                                nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status"))})
 101                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 102                         nsi_ready = False
 103
 104                 if nsrs_detailed_list != nsrs_detailed_list_old:
 105                     nsir_admin["nsrs-detailed-list"] = nsrs_detailed_list
 106                     nsrs_detailed_list_old = nsrs_detailed_list
 107                     db_nsir_update["_admin"] = nsir_admin
 108                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 109
 110                 if nsi_ready:
 111                     step = "Network Slice Instance is ready. nsi_id={}".format(nsir_id)
 112                     for items in nsrs_detailed_list:
 113                         if "FAILED" in items.values():
 114                             raise LcmException("Error deploying NSI: {}".format(nsir_id))
 115                     break
 116
 117                 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
 118                 await asyncio.sleep(5, loop=self.loop)
 119                 deployment_timeout -= 5
 120
 121             if deployment_timeout <= 0:
 122                 raise LcmException("Timeout waiting nsi to be ready. nsi_id={}".format(nsir_id))
 123
 124             db_nsir_update["operational-status"] = "running"
 125             db_nsir_update["detailed-status"] = "done"
 126             db_nsir_update["config-status"] = "configured"
 127             db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 128             db_nsilcmop_update["statusEnteredTime"] = time()
 129             db_nsilcmop_update["detailed-status"] = "done"
 130             return
 131
 132         except (LcmException, DbException) as e:
 133             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 134             exc = e
 135         except asyncio.CancelledError:
 136             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 137             exc = "Operation was cancelled"
 138         except Exception as e:
 139             exc = traceback.format_exc()
 140             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 141                                  exc_info=True)
 142         finally:
 143             if exc:
 144                 if db_nsir:
 145                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 146                     db_nsir_update["operational-status"] = "failed"
 147                 if db_nsilcmop:
 148                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 149                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 150                     db_nsilcmop_update["statusEnteredTime"] = time()
 151             if db_nsir:
 152                 db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
 153                 db_nsir_update["_admin.nsilcmop"] = None
 154                 self.update_db_2("nsis", nsir_id, db_nsir_update)
 155             if db_nsilcmop:
 156
 157                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 158             if nsilcmop_operation_state:
 159                 try:
 160                     await self.msg.aiowrite("nsi", "instantiated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 161                                                                     "operationState": nsilcmop_operation_state})
 162                 except Exception as e:
 163                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 164             self.logger.debug(logging_text + "Exit")
 165             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
 166
 167     async def terminate(self, nsir_id, nsilcmop_id):
 168         logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
 169         self.logger.debug(logging_text + "Enter")
 170         exc = None
 171         db_nsir = None
 172         db_nsilcmop = None
 173         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
 174         db_nsilcmop_update = {}
 175         nsilcmop_operation_state = None
 176         try:
 177             step = "Getting nsir={} from db".format(nsir_id)
 178             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 179             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 180             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 181
 182             # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
 183             # CASE: Instance was terminated but there is a second request to terminate the instance
 184             if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
 185                 return
 186
 187             # Slice status Terminating
 188             db_nsir_update["operational-status"] = "terminating"
 189             db_nsir_update["config-status"] = "terminating"
 190             self.update_db_2("nsis", nsir_id, db_nsir_update)
 191
 192             # look if previous tasks is in process
 193             task_name, task_dependency = self.lcm_tasks.lookfor_related("nsi", nsir_id, nsilcmop_id)
 194             if task_dependency:
 195                 step = db_nsilcmop_update["detailed-status"] = \
 196                     "Waiting for related tasks to be completed: {}".format(task_name)
 197                 self.logger.debug(logging_text + step)
 198                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 199                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 200                 if pending:
 201                     raise LcmException("Timeout waiting related tasks to be completed")
 202
 203             # Gets the list to keep track of network service records status in the netslice
 204             nsir_admin = db_nsir["_admin"]
 205             nsrs_detailed_list = []
 206
 207             # Iterate over the network services operation ids to terminate NSs
 208             # TODO: (future improvement) look another way check the tasks instead of keep asking
 209             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 210             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 211             # ns_tasks = []
 212             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 213             for nslcmop_id in nslcmop_ids:
 214                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 215                 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
 216                 task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
 217                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 218
 219             # Wait until Network Slice is terminated
 220             step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(nsir_id)
 221             nsrs_detailed_list_old = None
 222             self.logger.debug(logging_text + step)
 223
 224             termination_timeout = 2 * 3600   # Two hours
 225             while termination_timeout > 0:
 226                 # Check ns termination status
 227                 nsi_ready = True
 228                 nsrs_detailed_list = []
 229                 for nslcmop_item in nslcmop_ids:
 230                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 231                     status = nslcmop["operationState"]
 232                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 233                     nsrs_detailed_list.append({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 234                                                "detailed-status":
 235                                                nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status"))})
 236                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 237                         nsi_ready = False
 238
 239                 if nsrs_detailed_list != nsrs_detailed_list_old:
 240                     nsir_admin["nsrs-detailed-list"] = nsrs_detailed_list
 241                     nsrs_detailed_list_old = nsrs_detailed_list
 242                     db_nsir_update["_admin"] = nsir_admin
 243                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 244
 245                 if nsi_ready:
 246                     step = "Network Slice Instance is terminated. nsi_id={}".format(nsir_id)
 247                     for items in nsrs_detailed_list:
 248                         if "FAILED" in items.values():
 249                             raise LcmException("Error terminating NSI: {}".format(nsir_id))
 250                     break
 251
 252                 await asyncio.sleep(5, loop=self.loop)
 253                 termination_timeout -= 5
 254
 255             if termination_timeout <= 0:
 256                 raise LcmException("Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id))
 257
 258             db_nsir_update["operational-status"] = "terminated"
 259             db_nsir_update["config-status"] = "configured"
 260             db_nsir_update["detailed-status"] = "done"
 261             db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 262             db_nsilcmop_update["statusEnteredTime"] = time()
 263             db_nsilcmop_update["detailed-status"] = "done"
 264             return
 265
 266         except (LcmException, DbException) as e:
 267             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 268             exc = e
 269         except asyncio.CancelledError:
 270             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 271             exc = "Operation was cancelled"
 272         except Exception as e:
 273             exc = traceback.format_exc()
 274             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 275                                  exc_info=True)
 276         finally:
 277             if exc:
 278                 if db_nsir:
 279                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 280                     db_nsir_update["operational-status"] = "failed"
 281                 if db_nsilcmop:
 282                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 283                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 284                     db_nsilcmop_update["statusEnteredTime"] = time()
 285             if db_nsir:
 286                 db_nsir_update["_admin.nsilcmop"] = None
 287                 db_nsir_update["_admin.nsiState"] = "TERMINATED"
 288                 self.update_db_2("nsis", nsir_id, db_nsir_update)
 289             if db_nsilcmop:
 290                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 291
 292             if nsilcmop_operation_state:
 293                 try:
 294                     await self.msg.aiowrite("nsi", "terminated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 295                                                                   "operationState": nsilcmop_operation_state})
 296                 except Exception as e:
 297                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 298             self.logger.debug(logging_text + "Exit")
 299             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")