osm_lcm/vim_sdn.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import yaml
  20 import asyncio
  21 import logging
  22 import logging.handlers
  23 from osm_lcm import ROclient
  24 from osm_lcm.lcm_utils import LcmException, LcmBase, deep_get
  25 from n2vc.k8s_helm_conn import K8sHelmConnector
  26 from n2vc.k8s_helm3_conn import K8sHelm3Connector
  27 from n2vc.k8s_juju_conn import K8sJujuConnector
  28 from n2vc.n2vc_juju_conn import N2VCJujuConnector
  29 from n2vc.exceptions import K8sException, N2VCException
  30 from osm_common.dbbase import DbException
  31 from copy import deepcopy
  32 from time import time
  33
  34 __author__ = "Alfonso Tierno"
  35
  36
  37 class VimLcm(LcmBase):
  38     # values that are encrypted at vim config because they are passwords
  39     vim_config_encrypted = {"1.1": ("admin_password", "nsx_password", "vcenter_password"),
  40                             "default": ("admin_password", "nsx_password", "vcenter_password", "vrops_password")}
  41
  42     def __init__(self, msg, lcm_tasks, config, loop):
  43         """
  44         Init, Connect to database, filesystem storage, and messaging
  45         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  46         :return: None
  47         """
  48
  49         self.logger = logging.getLogger('lcm.vim')
  50         self.loop = loop
  51         self.lcm_tasks = lcm_tasks
  52         self.ro_config = config["ro_config"]
  53
  54         super().__init__(msg, self.logger)
  55
  56     async def create(self, vim_content, order_id):
  57
  58         # HA tasks and backward compatibility:
  59         # If 'vim_content' does not include 'op_id', we a running a legacy NBI version.
  60         # In such a case, HA is not supported by NBI, 'op_id' is None, and lock_HA() will do nothing.
  61         # Register 'create' task here for related future HA operations
  62         op_id = vim_content.pop('op_id', None)
  63         if not self.lcm_tasks.lock_HA('vim', 'create', op_id):
  64             return
  65
  66         vim_id = vim_content["_id"]
  67         logging_text = "Task vim_create={} ".format(vim_id)
  68         self.logger.debug(logging_text + "Enter")
  69
  70         db_vim = None
  71         db_vim_update = {}
  72         exc = None
  73         RO_sdn_id = None
  74         try:
  75             step = "Getting vim-id='{}' from db".format(vim_id)
  76             db_vim = self.db.get_one("vim_accounts", {"_id": vim_id})
  77             if vim_content.get("config") and vim_content["config"].get("sdn-controller"):
  78                 step = "Getting sdn-controller-id='{}' from db".format(vim_content["config"]["sdn-controller"])
  79                 db_sdn = self.db.get_one("sdns", {"_id": vim_content["config"]["sdn-controller"]})
  80
  81                 # If the VIM account has an associated SDN account, also
  82                 # wait for any previous tasks in process for the SDN
  83                 await self.lcm_tasks.waitfor_related_HA('sdn', 'ANY', db_sdn["_id"])
  84
  85                 if db_sdn.get("_admin") and db_sdn["_admin"].get("deployed") and db_sdn["_admin"]["deployed"].get("RO"):
  86                     RO_sdn_id = db_sdn["_admin"]["deployed"]["RO"]
  87                 else:
  88                     raise LcmException("sdn-controller={} is not available. Not deployed at RO".format(
  89                         vim_content["config"]["sdn-controller"]))
  90
  91             step = "Creating vim at RO"
  92             db_vim_update["_admin.deployed.RO"] = None
  93             db_vim_update["_admin.detailed-status"] = step
  94             self.update_db_2("vim_accounts", vim_id, db_vim_update)
  95             RO = ROclient.ROClient(self.loop, **self.ro_config)
  96             vim_RO = deepcopy(vim_content)
  97             vim_RO.pop("_id", None)
  98             vim_RO.pop("_admin", None)
  99             schema_version = vim_RO.pop("schema_version", None)
 100             vim_RO.pop("schema_type", None)
 101             vim_RO.pop("vim_tenant_name", None)
 102             vim_RO["type"] = vim_RO.pop("vim_type")
 103             vim_RO.pop("vim_user", None)
 104             vim_RO.pop("vim_password", None)
 105             if RO_sdn_id:
 106                 vim_RO["config"]["sdn-controller"] = RO_sdn_id
 107             desc = await RO.create("vim", descriptor=vim_RO)
 108             RO_vim_id = desc["uuid"]
 109             db_vim_update["_admin.deployed.RO"] = RO_vim_id
 110             self.logger.debug(logging_text + "VIM created at RO_vim_id={}".format(RO_vim_id))
 111
 112             step = "Creating vim_account at RO"
 113             db_vim_update["_admin.detailed-status"] = step
 114             self.update_db_2("vim_accounts", vim_id, db_vim_update)
 115
 116             if vim_content.get("vim_password"):
 117                 vim_content["vim_password"] = self.db.decrypt(vim_content["vim_password"],
 118                                                               schema_version=schema_version,
 119                                                               salt=vim_id)
 120             vim_account_RO = {"vim_tenant_name": vim_content["vim_tenant_name"],
 121                               "vim_username": vim_content["vim_user"],
 122                               "vim_password": vim_content["vim_password"]
 123                               }
 124             if vim_RO.get("config"):
 125                 vim_account_RO["config"] = vim_RO["config"]
 126                 if "sdn-controller" in vim_account_RO["config"]:
 127                     del vim_account_RO["config"]["sdn-controller"]
 128                 if "sdn-port-mapping" in vim_account_RO["config"]:
 129                     del vim_account_RO["config"]["sdn-port-mapping"]
 130                 vim_config_encrypted_keys = self.vim_config_encrypted.get(schema_version) or \
 131                     self.vim_config_encrypted.get("default")
 132                 for p in vim_config_encrypted_keys:
 133                     if vim_account_RO["config"].get(p):
 134                         vim_account_RO["config"][p] = self.db.decrypt(vim_account_RO["config"][p],
 135                                                                       schema_version=schema_version,
 136                                                                       salt=vim_id)
 137
 138             desc = await RO.attach("vim_account", RO_vim_id, descriptor=vim_account_RO)
 139             db_vim_update["_admin.deployed.RO-account"] = desc["uuid"]
 140             db_vim_update["_admin.operationalState"] = "ENABLED"
 141             db_vim_update["_admin.detailed-status"] = "Done"
 142             # Mark the VIM 'create' HA task as successful
 143             operation_state = 'COMPLETED'
 144             operation_details = 'Done'
 145
 146             self.logger.debug(logging_text + "Exit Ok VIM account created at RO_vim_account_id={}".format(desc["uuid"]))
 147             return
 148
 149         except (ROclient.ROClientException, DbException, asyncio.CancelledError) as e:
 150             self.logger.error(logging_text + "Exit Exception {}".format(e))
 151             exc = e
 152         except Exception as e:
 153             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
 154             exc = e
 155         finally:
 156             if exc and db_vim:
 157                 db_vim_update["_admin.operationalState"] = "ERROR"
 158                 db_vim_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
 159                 # Mark the VIM 'create' HA task as erroneous
 160                 operation_state = 'FAILED'
 161                 operation_details = "ERROR {}: {}".format(step, exc)
 162             try:
 163                 if db_vim_update:
 164                     self.update_db_2("vim_accounts", vim_id, db_vim_update)
 165                 # Register the VIM 'create' HA task either
 166                 # succesful or erroneous, or do nothing (if legacy NBI)
 167                 self.lcm_tasks.unlock_HA('vim', 'create', op_id,
 168                                          operationState=operation_state,
 169                                          detailed_status=operation_details)
 170             except DbException as e:
 171                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 172
 173             self.lcm_tasks.remove("vim_account", vim_id, order_id)
 174
 175     async def edit(self, vim_content, order_id):
 176
 177         # HA tasks and backward compatibility:
 178         # If 'vim_content' does not include 'op_id', we a running a legacy NBI version.
 179         # In such a case, HA is not supported by NBI, and the HA check always returns True
 180         op_id = vim_content.pop('op_id', None)
 181         if not self.lcm_tasks.lock_HA('vim', 'edit', op_id):
 182             return
 183
 184         vim_id = vim_content["_id"]
 185         logging_text = "Task vim_edit={} ".format(vim_id)
 186         self.logger.debug(logging_text + "Enter")
 187
 188         db_vim = None
 189         exc = None
 190         RO_sdn_id = None
 191         RO_vim_id = None
 192         db_vim_update = {}
 193         step = "Getting vim-id='{}' from db".format(vim_id)
 194         try:
 195             # wait for any previous tasks in process
 196             await self.lcm_tasks.waitfor_related_HA('vim', 'edit', op_id)
 197
 198             db_vim = self.db.get_one("vim_accounts", {"_id": vim_id})
 199
 200             if db_vim.get("_admin") and db_vim["_admin"].get("deployed") and db_vim["_admin"]["deployed"].get("RO"):
 201                 if vim_content.get("config") and vim_content["config"].get("sdn-controller"):
 202                     step = "Getting sdn-controller-id='{}' from db".format(vim_content["config"]["sdn-controller"])
 203                     db_sdn = self.db.get_one("sdns", {"_id": vim_content["config"]["sdn-controller"]})
 204
 205                     # If the VIM account has an associated SDN account, also
 206                     # wait for any previous tasks in process for the SDN
 207                     await self.lcm_tasks.waitfor_related_HA('sdn', 'ANY', db_sdn["_id"])
 208
 209                     if db_sdn.get("_admin") and db_sdn["_admin"].get("deployed") and db_sdn["_admin"]["deployed"].get(
 210                             "RO"):
 211                         RO_sdn_id = db_sdn["_admin"]["deployed"]["RO"]
 212                     else:
 213                         raise LcmException("sdn-controller={} is not available. Not deployed at RO".format(
 214                             vim_content["config"]["sdn-controller"]))
 215
 216                 RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 217                 step = "Editing vim at RO"
 218                 RO = ROclient.ROClient(self.loop, **self.ro_config)
 219                 vim_RO = deepcopy(vim_content)
 220                 vim_RO.pop("_id", None)
 221                 vim_RO.pop("_admin", None)
 222                 schema_version = vim_RO.pop("schema_version", None)
 223                 vim_RO.pop("schema_type", None)
 224                 vim_RO.pop("vim_tenant_name", None)
 225                 if "vim_type" in vim_RO:
 226                     vim_RO["type"] = vim_RO.pop("vim_type")
 227                 vim_RO.pop("vim_user", None)
 228                 vim_RO.pop("vim_password", None)
 229                 if RO_sdn_id:
 230                     vim_RO["config"]["sdn-controller"] = RO_sdn_id
 231                 # TODO make a deep update of sdn-port-mapping
 232                 if vim_RO:
 233                     await RO.edit("vim", RO_vim_id, descriptor=vim_RO)
 234
 235                 step = "Editing vim-account at RO tenant"
 236                 vim_account_RO = {}
 237                 if "config" in vim_content:
 238                     if "sdn-controller" in vim_content["config"]:
 239                         del vim_content["config"]["sdn-controller"]
 240                     if "sdn-port-mapping" in vim_content["config"]:
 241                         del vim_content["config"]["sdn-port-mapping"]
 242                     if not vim_content["config"]:
 243                         del vim_content["config"]
 244                 if "vim_tenant_name" in vim_content:
 245                     vim_account_RO["vim_tenant_name"] = vim_content["vim_tenant_name"]
 246                 if "vim_password" in vim_content:
 247                     vim_account_RO["vim_password"] = vim_content["vim_password"]
 248                 if vim_content.get("vim_password"):
 249                     vim_account_RO["vim_password"] = self.db.decrypt(vim_content["vim_password"],
 250                                                                      schema_version=schema_version,
 251                                                                      salt=vim_id)
 252                 if "config" in vim_content:
 253                     vim_account_RO["config"] = vim_content["config"]
 254                 if vim_content.get("config"):
 255                     vim_config_encrypted_keys = self.vim_config_encrypted.get(schema_version) or \
 256                         self.vim_config_encrypted.get("default")
 257                     for p in vim_config_encrypted_keys:
 258                         if vim_content["config"].get(p):
 259                             vim_account_RO["config"][p] = self.db.decrypt(vim_content["config"][p],
 260                                                                           schema_version=schema_version,
 261                                                                           salt=vim_id)
 262
 263                 if "vim_user" in vim_content:
 264                     vim_content["vim_username"] = vim_content["vim_user"]
 265                 # vim_account must be edited always even if empty in order to ensure changes are translated to RO
 266                 # vim_thread. RO will remove and relaunch a new thread for this vim_account
 267                 await RO.edit("vim_account", RO_vim_id, descriptor=vim_account_RO)
 268                 db_vim_update["_admin.operationalState"] = "ENABLED"
 269                 # Mark the VIM 'edit' HA task as successful
 270             operation_state = 'COMPLETED'
 271             operation_details = 'Done'
 272
 273             self.logger.debug(logging_text + "Exit Ok RO_vim_id={}".format(RO_vim_id))
 274             return
 275
 276         except (ROclient.ROClientException, DbException, asyncio.CancelledError) as e:
 277             self.logger.error(logging_text + "Exit Exception {}".format(e))
 278             exc = e
 279         except Exception as e:
 280             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
 281             exc = e
 282         finally:
 283             if exc and db_vim:
 284                 db_vim_update["_admin.operationalState"] = "ERROR"
 285                 db_vim_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
 286                 # Mark the VIM 'edit' HA task as erroneous
 287                 operation_state = 'FAILED'
 288                 operation_details = "ERROR {}: {}".format(step, exc)
 289             try:
 290                 if db_vim_update:
 291                     self.update_db_2("vim_accounts", vim_id, db_vim_update)
 292                 # Register the VIM 'edit' HA task either
 293                 # succesful or erroneous, or do nothing (if legacy NBI)
 294                 self.lcm_tasks.unlock_HA('vim', 'edit', op_id,
 295                                          operationState=operation_state,
 296                                          detailed_status=operation_details)
 297             except DbException as e:
 298                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 299
 300             self.lcm_tasks.remove("vim_account", vim_id, order_id)
 301
 302     async def delete(self, vim_content, order_id):
 303
 304         # HA tasks and backward compatibility:
 305         # If 'vim_content' does not include 'op_id', we a running a legacy NBI version.
 306         # In such a case, HA is not supported by NBI, and the HA check always returns True
 307         op_id = vim_content.pop('op_id', None)
 308         if not self.lcm_tasks.lock_HA('vim', 'delete', op_id):
 309             return
 310
 311         vim_id = vim_content["_id"]
 312         logging_text = "Task vim_delete={} ".format(vim_id)
 313         self.logger.debug(logging_text + "Enter")
 314
 315         db_vim = None
 316         db_vim_update = {}
 317         exc = None
 318         step = "Getting vim from db"
 319         try:
 320             # wait for any previous tasks in process
 321             await self.lcm_tasks.waitfor_related_HA('vim', 'delete', op_id)
 322             if not self.ro_config.get("ng"):
 323                 db_vim = self.db.get_one("vim_accounts", {"_id": vim_id})
 324                 if db_vim.get("_admin") and db_vim["_admin"].get("deployed") and db_vim["_admin"]["deployed"].get("RO"):
 325                     RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 326                     RO = ROclient.ROClient(self.loop, **self.ro_config)
 327                     step = "Detaching vim from RO tenant"
 328                     try:
 329                         await RO.detach("vim_account", RO_vim_id)
 330                     except ROclient.ROClientException as e:
 331                         if e.http_code == 404:  # not found
 332                             self.logger.debug(logging_text + "RO_vim_id={} already detached".format(RO_vim_id))
 333                         else:
 334                             raise
 335
 336                     step = "Deleting vim from RO"
 337                     try:
 338                         await RO.delete("vim", RO_vim_id)
 339                     except ROclient.ROClientException as e:
 340                         if e.http_code == 404:  # not found
 341                             self.logger.debug(logging_text + "RO_vim_id={} already deleted".format(RO_vim_id))
 342                         else:
 343                             raise
 344                 else:
 345                     # nothing to delete
 346                     self.logger.debug(logging_text + "Nothing to remove at RO")
 347             self.db.del_one("vim_accounts", {"_id": vim_id})
 348             db_vim = None
 349             self.logger.debug(logging_text + "Exit Ok")
 350             return
 351
 352         except (ROclient.ROClientException, DbException, asyncio.CancelledError) as e:
 353             self.logger.error(logging_text + "Exit Exception {}".format(e))
 354             exc = e
 355         except Exception as e:
 356             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
 357             exc = e
 358         finally:
 359             self.lcm_tasks.remove("vim_account", vim_id, order_id)
 360             if exc and db_vim:
 361                 db_vim_update["_admin.operationalState"] = "ERROR"
 362                 db_vim_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
 363                 # Mark the VIM 'delete' HA task as erroneous
 364                 operation_state = 'FAILED'
 365                 operation_details = "ERROR {}: {}".format(step, exc)
 366                 self.lcm_tasks.unlock_HA('vim', 'delete', op_id,
 367                                          operationState=operation_state,
 368                                          detailed_status=operation_details)
 369             try:
 370                 if db_vim and db_vim_update:
 371                     self.update_db_2("vim_accounts", vim_id, db_vim_update)
 372                 # If the VIM 'delete' HA task was succesful, the DB entry has been deleted,
 373                 # which means that there is nowhere to register this task, so do nothing here.
 374             except DbException as e:
 375                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 376             self.lcm_tasks.remove("vim_account", vim_id, order_id)
 377
 378
 379 class WimLcm(LcmBase):
 380     # values that are encrypted at wim config because they are passwords
 381     wim_config_encrypted = ()
 382
 383     def __init__(self, msg, lcm_tasks, config, loop):
 384         """
 385         Init, Connect to database, filesystem storage, and messaging
 386         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
 387         :return: None
 388         """
 389
 390         self.logger = logging.getLogger('lcm.vim')
 391         self.loop = loop
 392         self.lcm_tasks = lcm_tasks
 393         self.ro_config = config["ro_config"]
 394
 395         super().__init__(msg, self.logger)
 396
 397     async def create(self, wim_content, order_id):
 398
 399         # HA tasks and backward compatibility:
 400         # If 'wim_content' does not include 'op_id', we a running a legacy NBI version.
 401         # In such a case, HA is not supported by NBI, 'op_id' is None, and lock_HA() will do nothing.
 402         # Register 'create' task here for related future HA operations
 403         op_id = wim_content.pop('op_id', None)
 404         self.lcm_tasks.lock_HA('wim', 'create', op_id)
 405
 406         wim_id = wim_content["_id"]
 407         logging_text = "Task wim_create={} ".format(wim_id)
 408         self.logger.debug(logging_text + "Enter")
 409
 410         db_wim = None
 411         db_wim_update = {}
 412         exc = None
 413         try:
 414             step = "Getting wim-id='{}' from db".format(wim_id)
 415             db_wim = self.db.get_one("wim_accounts", {"_id": wim_id})
 416             db_wim_update["_admin.deployed.RO"] = None
 417
 418             step = "Creating wim at RO"
 419             db_wim_update["_admin.detailed-status"] = step
 420             self.update_db_2("wim_accounts", wim_id, db_wim_update)
 421             RO = ROclient.ROClient(self.loop, **self.ro_config)
 422             wim_RO = deepcopy(wim_content)
 423             wim_RO.pop("_id", None)
 424             wim_RO.pop("_admin", None)
 425             schema_version = wim_RO.pop("schema_version", None)
 426             wim_RO.pop("schema_type", None)
 427             wim_RO.pop("wim_tenant_name", None)
 428             wim_RO["type"] = wim_RO.pop("wim_type")
 429             wim_RO.pop("wim_user", None)
 430             wim_RO.pop("wim_password", None)
 431             desc = await RO.create("wim", descriptor=wim_RO)
 432             RO_wim_id = desc["uuid"]
 433             db_wim_update["_admin.deployed.RO"] = RO_wim_id
 434             self.logger.debug(logging_text + "WIM created at RO_wim_id={}".format(RO_wim_id))
 435
 436             step = "Creating wim_account at RO"
 437             db_wim_update["_admin.detailed-status"] = step
 438             self.update_db_2("wim_accounts", wim_id, db_wim_update)
 439
 440             if wim_content.get("wim_password"):
 441                 wim_content["wim_password"] = self.db.decrypt(wim_content["wim_password"],
 442                                                               schema_version=schema_version,
 443                                                               salt=wim_id)
 444             wim_account_RO = {"name": wim_content["name"],
 445                               "user": wim_content["user"],
 446                               "password": wim_content["password"]
 447                               }
 448             if wim_RO.get("config"):
 449                 wim_account_RO["config"] = wim_RO["config"]
 450                 if "wim_port_mapping" in wim_account_RO["config"]:
 451                     del wim_account_RO["config"]["wim_port_mapping"]
 452                 for p in self.wim_config_encrypted:
 453                     if wim_account_RO["config"].get(p):
 454                         wim_account_RO["config"][p] = self.db.decrypt(wim_account_RO["config"][p],
 455                                                                       schema_version=schema_version,
 456                                                                       salt=wim_id)
 457
 458             desc = await RO.attach("wim_account", RO_wim_id, descriptor=wim_account_RO)
 459             db_wim_update["_admin.deployed.RO-account"] = desc["uuid"]
 460             db_wim_update["_admin.operationalState"] = "ENABLED"
 461             db_wim_update["_admin.detailed-status"] = "Done"
 462             # Mark the WIM 'create' HA task as successful
 463             operation_state = 'COMPLETED'
 464             operation_details = 'Done'
 465
 466             self.logger.debug(logging_text + "Exit Ok WIM account created at RO_wim_account_id={}".format(desc["uuid"]))
 467             return
 468
 469         except (ROclient.ROClientException, DbException, asyncio.CancelledError) as e:
 470             self.logger.error(logging_text + "Exit Exception {}".format(e))
 471             exc = e
 472         except Exception as e:
 473             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
 474             exc = e
 475         finally:
 476             if exc and db_wim:
 477                 db_wim_update["_admin.operationalState"] = "ERROR"
 478                 db_wim_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
 479                 # Mark the WIM 'create' HA task as erroneous
 480                 operation_state = 'FAILED'
 481                 operation_details = "ERROR {}: {}".format(step, exc)
 482             try:
 483                 if db_wim_update:
 484                     self.update_db_2("wim_accounts", wim_id, db_wim_update)
 485                 # Register the WIM 'create' HA task either
 486                 # succesful or erroneous, or do nothing (if legacy NBI)
 487                 self.lcm_tasks.unlock_HA('wim', 'create', op_id,
 488                                          operationState=operation_state,
 489                                          detailed_status=operation_details)
 490             except DbException as e:
 491                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 492             self.lcm_tasks.remove("wim_account", wim_id, order_id)
 493
 494     async def edit(self, wim_content, order_id):
 495
 496         # HA tasks and backward compatibility:
 497         # If 'wim_content' does not include 'op_id', we a running a legacy NBI version.
 498         # In such a case, HA is not supported by NBI, and the HA check always returns True
 499         op_id = wim_content.pop('op_id', None)
 500         if not self.lcm_tasks.lock_HA('wim', 'edit', op_id):
 501             return
 502
 503         wim_id = wim_content["_id"]
 504         logging_text = "Task wim_edit={} ".format(wim_id)
 505         self.logger.debug(logging_text + "Enter")
 506
 507         db_wim = None
 508         exc = None
 509         RO_wim_id = None
 510         db_wim_update = {}
 511         step = "Getting wim-id='{}' from db".format(wim_id)
 512         try:
 513             # wait for any previous tasks in process
 514             await self.lcm_tasks.waitfor_related_HA('wim', 'edit', op_id)
 515
 516             db_wim = self.db.get_one("wim_accounts", {"_id": wim_id})
 517
 518             if db_wim.get("_admin") and db_wim["_admin"].get("deployed") and db_wim["_admin"]["deployed"].get("RO"):
 519
 520                 RO_wim_id = db_wim["_admin"]["deployed"]["RO"]
 521                 step = "Editing wim at RO"
 522                 RO = ROclient.ROClient(self.loop, **self.ro_config)
 523                 wim_RO = deepcopy(wim_content)
 524                 wim_RO.pop("_id", None)
 525                 wim_RO.pop("_admin", None)
 526                 schema_version = wim_RO.pop("schema_version", None)
 527                 wim_RO.pop("schema_type", None)
 528                 wim_RO.pop("wim_tenant_name", None)
 529                 if "wim_type" in wim_RO:
 530                     wim_RO["type"] = wim_RO.pop("wim_type")
 531                 wim_RO.pop("wim_user", None)
 532                 wim_RO.pop("wim_password", None)
 533                 # TODO make a deep update of wim_port_mapping
 534                 if wim_RO:
 535                     await RO.edit("wim", RO_wim_id, descriptor=wim_RO)
 536
 537                 step = "Editing wim-account at RO tenant"
 538                 wim_account_RO = {}
 539                 if "config" in wim_content:
 540                     if "wim_port_mapping" in wim_content["config"]:
 541                         del wim_content["config"]["wim_port_mapping"]
 542                     if not wim_content["config"]:
 543                         del wim_content["config"]
 544                 if "wim_tenant_name" in wim_content:
 545                     wim_account_RO["wim_tenant_name"] = wim_content["wim_tenant_name"]
 546                 if "wim_password" in wim_content:
 547                     wim_account_RO["wim_password"] = wim_content["wim_password"]
 548                 if wim_content.get("wim_password"):
 549                     wim_account_RO["wim_password"] = self.db.decrypt(wim_content["wim_password"],
 550                                                                      schema_version=schema_version,
 551                                                                      salt=wim_id)
 552                 if "config" in wim_content:
 553                     wim_account_RO["config"] = wim_content["config"]
 554                 if wim_content.get("config"):
 555                     for p in self.wim_config_encrypted:
 556                         if wim_content["config"].get(p):
 557                             wim_account_RO["config"][p] = self.db.decrypt(wim_content["config"][p],
 558                                                                           schema_version=schema_version,
 559                                                                           salt=wim_id)
 560
 561                 if "wim_user" in wim_content:
 562                     wim_content["wim_username"] = wim_content["wim_user"]
 563                 # wim_account must be edited always even if empty in order to ensure changes are translated to RO
 564                 # wim_thread. RO will remove and relaunch a new thread for this wim_account
 565                 await RO.edit("wim_account", RO_wim_id, descriptor=wim_account_RO)
 566                 db_wim_update["_admin.operationalState"] = "ENABLED"
 567                 # Mark the WIM 'edit' HA task as successful
 568             operation_state = 'COMPLETED'
 569             operation_details = 'Done'
 570
 571             self.logger.debug(logging_text + "Exit Ok RO_wim_id={}".format(RO_wim_id))
 572             return
 573
 574         except (ROclient.ROClientException, DbException, asyncio.CancelledError) as e:
 575             self.logger.error(logging_text + "Exit Exception {}".format(e))
 576             exc = e
 577         except Exception as e:
 578             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
 579             exc = e
 580         finally:
 581             if exc and db_wim:
 582                 db_wim_update["_admin.operationalState"] = "ERROR"
 583                 db_wim_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
 584                 # Mark the WIM 'edit' HA task as erroneous
 585                 operation_state = 'FAILED'
 586                 operation_details = "ERROR {}: {}".format(step, exc)
 587             try:
 588                 if db_wim_update:
 589                     self.update_db_2("wim_accounts", wim_id, db_wim_update)
 590                 # Register the WIM 'edit' HA task either
 591                 # succesful or erroneous, or do nothing (if legacy NBI)
 592                 self.lcm_tasks.unlock_HA('wim', 'edit', op_id,
 593                                          operationState=operation_state,
 594                                          detailed_status=operation_details)
 595             except DbException as e:
 596                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 597             self.lcm_tasks.remove("wim_account", wim_id, order_id)
 598
 599     async def delete(self, wim_content, order_id):
 600
 601         # HA tasks and backward compatibility:
 602         # If 'vim_content' does not include 'op_id', we a running a legacy NBI version.
 603         # In such a case, HA is not supported by NBI, and the HA check always returns True
 604         op_id = wim_content.pop('op_id', None)
 605         if not self.lcm_tasks.lock_HA('wim', 'delete', op_id):
 606             return
 607
 608         wim_id = wim_content["_id"]
 609         logging_text = "Task wim_delete={} ".format(wim_id)
 610         self.logger.debug(logging_text + "Enter")
 611
 612         db_wim = None
 613         db_wim_update = {}
 614         exc = None
 615         step = "Getting wim from db"
 616         try:
 617             # wait for any previous tasks in process
 618             await self.lcm_tasks.waitfor_related_HA('wim', 'delete', op_id)
 619
 620             db_wim = self.db.get_one("wim_accounts", {"_id": wim_id})
 621             if db_wim.get("_admin") and db_wim["_admin"].get("deployed") and db_wim["_admin"]["deployed"].get("RO"):
 622                 RO_wim_id = db_wim["_admin"]["deployed"]["RO"]
 623                 RO = ROclient.ROClient(self.loop, **self.ro_config)
 624                 step = "Detaching wim from RO tenant"
 625                 try:
 626                     await RO.detach("wim_account", RO_wim_id)
 627                 except ROclient.ROClientException as e:
 628                     if e.http_code == 404:  # not found
 629                         self.logger.debug(logging_text + "RO_wim_id={} already detached".format(RO_wim_id))
 630                     else:
 631                         raise
 632
 633                 step = "Deleting wim from RO"
 634                 try:
 635                     await RO.delete("wim", RO_wim_id)
 636                 except ROclient.ROClientException as e:
 637                     if e.http_code == 404:  # not found
 638                         self.logger.debug(logging_text + "RO_wim_id={} already deleted".format(RO_wim_id))
 639                     else:
 640                         raise
 641             else:
 642                 # nothing to delete
 643                 self.logger.error(logging_text + "Nohing to remove at RO")
 644             self.db.del_one("wim_accounts", {"_id": wim_id})
 645             db_wim = None
 646             self.logger.debug(logging_text + "Exit Ok")
 647             return
 648
 649         except (ROclient.ROClientException, DbException, asyncio.CancelledError) as e:
 650             self.logger.error(logging_text + "Exit Exception {}".format(e))
 651             exc = e
 652         except Exception as e:
 653             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
 654             exc = e
 655         finally:
 656             self.lcm_tasks.remove("wim_account", wim_id, order_id)
 657             if exc and db_wim:
 658                 db_wim_update["_admin.operationalState"] = "ERROR"
 659                 db_wim_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
 660                 # Mark the WIM 'delete' HA task as erroneous
 661                 operation_state = 'FAILED'
 662                 operation_details = "ERROR {}: {}".format(step, exc)
 663                 self.lcm_tasks.unlock_HA('wim', 'delete', op_id,
 664                                          operationState=operation_state,
 665                                          detailed_status=operation_details)
 666             try:
 667                 if db_wim and db_wim_update:
 668                     self.update_db_2("wim_accounts", wim_id, db_wim_update)
 669                 # If the WIM 'delete' HA task was succesful, the DB entry has been deleted,
 670                 # which means that there is nowhere to register this task, so do nothing here.
 671             except DbException as e:
 672                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 673             self.lcm_tasks.remove("wim_account", wim_id, order_id)
 674
 675
 676 class SdnLcm(LcmBase):
 677
 678     def __init__(self, msg, lcm_tasks, config, loop):
 679         """
 680         Init, Connect to database, filesystem storage, and messaging
 681         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
 682         :return: None
 683         """
 684
 685         self.logger = logging.getLogger('lcm.sdn')
 686         self.loop = loop
 687         self.lcm_tasks = lcm_tasks
 688         self.ro_config = config["ro_config"]
 689
 690         super().__init__(msg, self.logger)
 691
 692     async def create(self, sdn_content, order_id):
 693
 694         # HA tasks and backward compatibility:
 695         # If 'sdn_content' does not include 'op_id', we a running a legacy NBI version.
 696         # In such a case, HA is not supported by NBI, 'op_id' is None, and lock_HA() will do nothing.
 697         # Register 'create' task here for related future HA operations
 698         op_id = sdn_content.pop('op_id', None)
 699         self.lcm_tasks.lock_HA('sdn', 'create', op_id)
 700
 701         sdn_id = sdn_content["_id"]
 702         logging_text = "Task sdn_create={} ".format(sdn_id)
 703         self.logger.debug(logging_text + "Enter")
 704
 705         db_sdn = None
 706         db_sdn_update = {}
 707         RO_sdn_id = None
 708         exc = None
 709         try:
 710             step = "Getting sdn from db"
 711             db_sdn = self.db.get_one("sdns", {"_id": sdn_id})
 712             db_sdn_update["_admin.deployed.RO"] = None
 713
 714             step = "Creating sdn at RO"
 715             db_sdn_update["_admin.detailed-status"] = step
 716             self.update_db_2("sdns", sdn_id, db_sdn_update)
 717
 718             RO = ROclient.ROClient(self.loop, **self.ro_config)
 719             sdn_RO = deepcopy(sdn_content)
 720             sdn_RO.pop("_id", None)
 721             sdn_RO.pop("_admin", None)
 722             schema_version = sdn_RO.pop("schema_version", None)
 723             sdn_RO.pop("schema_type", None)
 724             sdn_RO.pop("description", None)
 725             if sdn_RO.get("password"):
 726                 sdn_RO["password"] = self.db.decrypt(sdn_RO["password"], schema_version=schema_version, salt=sdn_id)
 727
 728             desc = await RO.create("sdn", descriptor=sdn_RO)
 729             RO_sdn_id = desc["uuid"]
 730             db_sdn_update["_admin.deployed.RO"] = RO_sdn_id
 731             db_sdn_update["_admin.operationalState"] = "ENABLED"
 732             self.logger.debug(logging_text + "Exit Ok RO_sdn_id={}".format(RO_sdn_id))
 733             # Mark the SDN 'create' HA task as successful
 734             operation_state = 'COMPLETED'
 735             operation_details = 'Done'
 736             return
 737
 738         except (ROclient.ROClientException, DbException, asyncio.CancelledError) as e:
 739             self.logger.error(logging_text + "Exit Exception {}".format(e))
 740             exc = e
 741         except Exception as e:
 742             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
 743             exc = e
 744         finally:
 745             if exc and db_sdn:
 746                 db_sdn_update["_admin.operationalState"] = "ERROR"
 747                 db_sdn_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
 748                 # Mark the SDN 'create' HA task as erroneous
 749                 operation_state = 'FAILED'
 750                 operation_details = "ERROR {}: {}".format(step, exc)
 751             try:
 752                 if db_sdn and db_sdn_update:
 753                     self.update_db_2("sdns", sdn_id, db_sdn_update)
 754                 # Register the SDN 'create' HA task either
 755                 # succesful or erroneous, or do nothing (if legacy NBI)
 756                 self.lcm_tasks.unlock_HA('sdn', 'create', op_id,
 757                                          operationState=operation_state,
 758                                          detailed_status=operation_details)
 759             except DbException as e:
 760                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 761             self.lcm_tasks.remove("sdn", sdn_id, order_id)
 762
 763     async def edit(self, sdn_content, order_id):
 764
 765         # HA tasks and backward compatibility:
 766         # If 'sdn_content' does not include 'op_id', we a running a legacy NBI version.
 767         # In such a case, HA is not supported by NBI, and the HA check always returns True
 768         op_id = sdn_content.pop('op_id', None)
 769         if not self.lcm_tasks.lock_HA('sdn', 'edit', op_id):
 770             return
 771
 772         sdn_id = sdn_content["_id"]
 773         logging_text = "Task sdn_edit={} ".format(sdn_id)
 774         self.logger.debug(logging_text + "Enter")
 775
 776         db_sdn = None
 777         db_sdn_update = {}
 778         exc = None
 779         step = "Getting sdn from db"
 780         try:
 781             # wait for any previous tasks in process
 782             await self.lcm_tasks.waitfor_related_HA('sdn', 'edit', op_id)
 783
 784             db_sdn = self.db.get_one("sdns", {"_id": sdn_id})
 785             RO_sdn_id = None
 786             if db_sdn.get("_admin") and db_sdn["_admin"].get("deployed") and db_sdn["_admin"]["deployed"].get("RO"):
 787                 RO_sdn_id = db_sdn["_admin"]["deployed"]["RO"]
 788                 RO = ROclient.ROClient(self.loop, **self.ro_config)
 789                 step = "Editing sdn at RO"
 790                 sdn_RO = deepcopy(sdn_content)
 791                 sdn_RO.pop("_id", None)
 792                 sdn_RO.pop("_admin", None)
 793                 schema_version = sdn_RO.pop("schema_version", None)
 794                 sdn_RO.pop("schema_type", None)
 795                 sdn_RO.pop("description", None)
 796                 if sdn_RO.get("password"):
 797                     sdn_RO["password"] = self.db.decrypt(sdn_RO["password"], schema_version=schema_version, salt=sdn_id)
 798                 if sdn_RO:
 799                     await RO.edit("sdn", RO_sdn_id, descriptor=sdn_RO)
 800                 db_sdn_update["_admin.operationalState"] = "ENABLED"
 801                 # Mark the SDN 'edit' HA task as successful
 802             operation_state = 'COMPLETED'
 803             operation_details = 'Done'
 804
 805             self.logger.debug(logging_text + "Exit Ok RO_sdn_id={}".format(RO_sdn_id))
 806             return
 807
 808         except (ROclient.ROClientException, DbException, asyncio.CancelledError) as e:
 809             self.logger.error(logging_text + "Exit Exception {}".format(e))
 810             exc = e
 811         except Exception as e:
 812             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
 813             exc = e
 814         finally:
 815             if exc and db_sdn:
 816                 db_sdn["_admin.operationalState"] = "ERROR"
 817                 db_sdn["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
 818                 # Mark the SDN 'edit' HA task as erroneous
 819                 operation_state = 'FAILED'
 820                 operation_details = "ERROR {}: {}".format(step, exc)
 821             try:
 822                 if db_sdn_update:
 823                     self.update_db_2("sdns", sdn_id, db_sdn_update)
 824                 # Register the SDN 'edit' HA task either
 825                 # succesful or erroneous, or do nothing (if legacy NBI)
 826                 self.lcm_tasks.unlock_HA('sdn', 'edit', op_id,
 827                                          operationState=operation_state,
 828                                          detailed_status=operation_details)
 829             except DbException as e:
 830                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 831             self.lcm_tasks.remove("sdn", sdn_id, order_id)
 832
 833     async def delete(self, sdn_content, order_id):
 834
 835         # HA tasks and backward compatibility:
 836         # If 'vim_content' does not include 'op_id', we a running a legacy NBI version.
 837         # In such a case, HA is not supported by NBI, and the HA check always returns True
 838         op_id = sdn_content.pop('op_id', None)
 839         if not self.lcm_tasks.lock_HA('sdn', 'delete', op_id):
 840             return
 841
 842         sdn_id = sdn_content["_id"]
 843         logging_text = "Task sdn_delete={} ".format(sdn_id)
 844         self.logger.debug(logging_text + "Enter")
 845
 846         db_sdn = None
 847         db_sdn_update = {}
 848         exc = None
 849         step = "Getting sdn from db"
 850         try:
 851             # wait for any previous tasks in process
 852             await self.lcm_tasks.waitfor_related_HA('sdn', 'delete', op_id)
 853
 854             db_sdn = self.db.get_one("sdns", {"_id": sdn_id})
 855             if db_sdn.get("_admin") and db_sdn["_admin"].get("deployed") and db_sdn["_admin"]["deployed"].get("RO"):
 856                 RO_sdn_id = db_sdn["_admin"]["deployed"]["RO"]
 857                 RO = ROclient.ROClient(self.loop, **self.ro_config)
 858                 step = "Deleting sdn from RO"
 859                 try:
 860                     await RO.delete("sdn", RO_sdn_id)
 861                 except ROclient.ROClientException as e:
 862                     if e.http_code == 404:  # not found
 863                         self.logger.debug(logging_text + "RO_sdn_id={} already deleted".format(RO_sdn_id))
 864                     else:
 865                         raise
 866             else:
 867                 # nothing to delete
 868                 self.logger.error(logging_text + "Skipping. There is not RO information at database")
 869             self.db.del_one("sdns", {"_id": sdn_id})
 870             db_sdn = None
 871             self.logger.debug("sdn_delete task sdn_id={} Exit Ok".format(sdn_id))
 872             return
 873
 874         except (ROclient.ROClientException, DbException, asyncio.CancelledError) as e:
 875             self.logger.error(logging_text + "Exit Exception {}".format(e))
 876             exc = e
 877         except Exception as e:
 878             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
 879             exc = e
 880         finally:
 881             if exc and db_sdn:
 882                 db_sdn["_admin.operationalState"] = "ERROR"
 883                 db_sdn["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
 884                 # Mark the SDN 'delete' HA task as erroneous
 885                 operation_state = 'FAILED'
 886                 operation_details = "ERROR {}: {}".format(step, exc)
 887                 self.lcm_tasks.unlock_HA('sdn', 'delete', op_id,
 888                                          operationState=operation_state,
 889                                          detailed_status=operation_details)
 890             try:
 891                 if db_sdn and db_sdn_update:
 892                     self.update_db_2("sdns", sdn_id, db_sdn_update)
 893                 # If the SDN 'delete' HA task was succesful, the DB entry has been deleted,
 894                 # which means that there is nowhere to register this task, so do nothing here.
 895             except DbException as e:
 896                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 897             self.lcm_tasks.remove("sdn", sdn_id, order_id)
 898
 899
 900 class K8sClusterLcm(LcmBase):
 901     timeout_create = 300
 902
 903     def __init__(self, msg, lcm_tasks, config, loop):
 904         """
 905         Init, Connect to database, filesystem storage, and messaging
 906         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
 907         :return: None
 908         """
 909
 910         self.logger = logging.getLogger('lcm.k8scluster')
 911         self.loop = loop
 912         self.lcm_tasks = lcm_tasks
 913         self.vca_config = config["VCA"]
 914
 915         super().__init__(msg, self.logger)
 916
 917         self.helm2_k8scluster = K8sHelmConnector(
 918             kubectl_command=self.vca_config.get("kubectlpath"),
 919             helm_command=self.vca_config.get("helmpath"),
 920             log=self.logger,
 921             on_update_db=None,
 922             db=self.db,
 923             fs=self.fs
 924         )
 925
 926         self.helm3_k8scluster = K8sHelm3Connector(
 927             kubectl_command=self.vca_config.get("kubectlpath"),
 928             helm_command=self.vca_config.get("helm3path"),
 929             fs=self.fs,
 930             log=self.logger,
 931             db=self.db,
 932             on_update_db=None
 933         )
 934
 935         self.juju_k8scluster = K8sJujuConnector(
 936             kubectl_command=self.vca_config.get("kubectlpath"),
 937             juju_command=self.vca_config.get("jujupath"),
 938             log=self.logger,
 939             loop=self.loop,
 940             on_update_db=None,
 941             db=self.db,
 942             fs=self.fs
 943         )
 944
 945         self.k8s_map = {
 946             "helm-chart": self.helm2_k8scluster,
 947             "helm-chart-v3": self.helm3_k8scluster,
 948             "juju-bundle": self.juju_k8scluster,
 949         }
 950
 951     async def create(self, k8scluster_content, order_id):
 952
 953         op_id = k8scluster_content.pop('op_id', None)
 954         if not self.lcm_tasks.lock_HA('k8scluster', 'create', op_id):
 955             return
 956
 957         k8scluster_id = k8scluster_content["_id"]
 958         logging_text = "Task k8scluster_create={} ".format(k8scluster_id)
 959         self.logger.debug(logging_text + "Enter")
 960
 961         db_k8scluster = None
 962         db_k8scluster_update = {}
 963         exc = None
 964         try:
 965             step = "Getting k8scluster-id='{}' from db".format(k8scluster_id)
 966             self.logger.debug(logging_text + step)
 967             db_k8scluster = self.db.get_one("k8sclusters", {"_id": k8scluster_id})
 968             self.db.encrypt_decrypt_fields(db_k8scluster.get("credentials"), 'decrypt', ['password', 'secret'],
 969                                            schema_version=db_k8scluster["schema_version"], salt=db_k8scluster["_id"])
 970             k8s_credentials = yaml.safe_dump(db_k8scluster.get("credentials"))
 971             pending_tasks = []
 972             task2name = {}
 973             init_target = deep_get(db_k8scluster, ("_admin", "init"))
 974             step = "Launching k8scluster init tasks"
 975             for task_name in ("helm-chart", "juju-bundle", "helm-chart-v3"):
 976                 if init_target and task_name not in init_target:
 977                     continue
 978                 task = asyncio.ensure_future(
 979                     self.k8s_map[task_name].init_env(
 980                         k8s_credentials,
 981                         reuse_cluster_uuid=k8scluster_id,
 982                         vca_id=db_k8scluster.get("vca_id"),
 983                     )
 984                 )
 985                 pending_tasks.append(task)
 986                 task2name[task] = task_name
 987
 988             error_text_list = []
 989             tasks_name_ok = []
 990             reached_timeout = False
 991             now = time()
 992
 993             while pending_tasks:
 994                 _timeout = max(1, self.timeout_create - (time() - now))  # ensure not negative with max
 995                 step = "Waiting for k8scluster init tasks"
 996                 done, pending_tasks = await asyncio.wait(pending_tasks, timeout=_timeout,
 997                                                          return_when=asyncio.FIRST_COMPLETED)
 998                 if not done:
 999                     # timeout. Set timeout is reached and process pending as if they hase been finished
1000                     done = pending_tasks
1001                     pending_tasks = None
1002                     reached_timeout = True
1003                 for task in done:
1004                     task_name = task2name[task]
1005                     if reached_timeout:
1006                         exc = "Timeout"
1007                     elif task.cancelled():
1008                         exc = "Cancelled"
1009                     else:
1010                         exc = task.exception()
1011
1012                     if exc:
1013                         error_text_list.append("Failing init {}: {}".format(task_name, exc))
1014                         db_k8scluster_update["_admin.{}.error_msg".format(task_name)] = str(exc)
1015                         db_k8scluster_update["_admin.{}.id".format(task_name)] = None
1016                         db_k8scluster_update["_admin.{}.operationalState".format(task_name)] = "ERROR"
1017                         self.logger.error(logging_text + "{} init fail: {}".format(task_name, exc),
1018                                           exc_info=not isinstance(exc, (N2VCException, str)))
1019                     else:
1020                         k8s_id, uninstall_sw = task.result()
1021                         tasks_name_ok.append(task_name)
1022                         self.logger.debug(logging_text + "{} init success. id={} created={}".format(
1023                             task_name, k8s_id, uninstall_sw))
1024                         db_k8scluster_update["_admin.{}.error_msg".format(task_name)] = None
1025                         db_k8scluster_update["_admin.{}.id".format(task_name)] = k8s_id
1026                         db_k8scluster_update["_admin.{}.created".format(task_name)] = uninstall_sw
1027                         db_k8scluster_update["_admin.{}.operationalState".format(task_name)] = "ENABLED"
1028                 # update database
1029                 step = "Updating database for " + task_name
1030                 self.update_db_2("k8sclusters", k8scluster_id, db_k8scluster_update)
1031             if tasks_name_ok:
1032                 operation_details = "ready for " + ", ".join(tasks_name_ok)
1033                 operation_state = "COMPLETED"
1034                 db_k8scluster_update["_admin.operationalState"] = "ENABLED" if not error_text_list else "DEGRADED"
1035                 operation_details += "; " + ";".join(error_text_list)
1036             else:
1037                 db_k8scluster_update["_admin.operationalState"] = "ERROR"
1038                 operation_state = "FAILED"
1039                 operation_details = ";".join(error_text_list)
1040             db_k8scluster_update["_admin.detailed-status"] = operation_details
1041             self.logger.debug(logging_text + "Done. Result: " + operation_state)
1042             exc = None
1043
1044         except Exception as e:
1045             if isinstance(e, (LcmException, DbException, K8sException, N2VCException, asyncio.CancelledError)):
1046                 self.logger.error(logging_text + "Exit Exception {}".format(e))
1047             else:
1048                 self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1049             exc = e
1050         finally:
1051             if exc and db_k8scluster:
1052                 db_k8scluster_update["_admin.operationalState"] = "ERROR"
1053                 db_k8scluster_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
1054                 operation_state = 'FAILED'
1055                 operation_details = "ERROR {}: {}".format(step, exc)
1056             try:
1057                 if db_k8scluster and db_k8scluster_update:
1058                     self.update_db_2("k8sclusters", k8scluster_id, db_k8scluster_update)
1059
1060                 # Register the operation and unlock
1061                 self.lcm_tasks.unlock_HA('k8scluster', 'create', op_id,
1062                                          operationState=operation_state,
1063                                          detailed_status=operation_details)
1064             except DbException as e:
1065                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1066             self.lcm_tasks.remove("k8scluster", k8scluster_id, order_id)
1067
1068     async def delete(self, k8scluster_content, order_id):
1069
1070         # HA tasks and backward compatibility:
1071         # If 'vim_content' does not include 'op_id', we a running a legacy NBI version.
1072         # In such a case, HA is not supported by NBI, 'op_id' is None, and lock_HA() will do nothing.
1073         # Register 'delete' task here for related future HA operations
1074         op_id = k8scluster_content.pop('op_id', None)
1075         if not self.lcm_tasks.lock_HA('k8scluster', 'delete', op_id):
1076             return
1077
1078         k8scluster_id = k8scluster_content["_id"]
1079         logging_text = "Task k8scluster_delete={} ".format(k8scluster_id)
1080         self.logger.debug(logging_text + "Enter")
1081
1082         db_k8scluster = None
1083         db_k8scluster_update = {}
1084         exc = None
1085         try:
1086             step = "Getting k8scluster='{}' from db".format(k8scluster_id)
1087             self.logger.debug(logging_text + step)
1088             db_k8scluster = self.db.get_one("k8sclusters", {"_id": k8scluster_id})
1089             k8s_hc_id = deep_get(db_k8scluster, ("_admin", "helm-chart", "id"))
1090             k8s_h3c_id = deep_get(db_k8scluster, ("_admin", "helm-chart-v3", "id"))
1091             k8s_jb_id = deep_get(db_k8scluster, ("_admin", "juju-bundle", "id"))
1092
1093             cluster_removed = True
1094             if k8s_jb_id:  # delete in reverse order of creation
1095                 step = "Removing juju-bundle '{}'".format(k8s_jb_id)
1096                 uninstall_sw = deep_get(db_k8scluster, ("_admin", "juju-bundle", "created")) or False
1097                 cluster_removed = await self.juju_k8scluster.reset(
1098                     cluster_uuid=k8s_jb_id,
1099                     uninstall_sw=uninstall_sw,
1100                     vca_id=db_k8scluster.get("vca_id"),
1101                 )
1102                 db_k8scluster_update["_admin.juju-bundle.id"] = None
1103                 db_k8scluster_update["_admin.juju-bundle.operationalState"] = "DISABLED"
1104
1105             if k8s_hc_id:
1106                 step = "Removing helm-chart '{}'".format(k8s_hc_id)
1107                 uninstall_sw = deep_get(db_k8scluster, ("_admin", "helm-chart", "created")) or False
1108                 cluster_removed = await self.helm2_k8scluster.reset(cluster_uuid=k8s_hc_id, uninstall_sw=uninstall_sw)
1109                 db_k8scluster_update["_admin.helm-chart.id"] = None
1110                 db_k8scluster_update["_admin.helm-chart.operationalState"] = "DISABLED"
1111
1112             if k8s_h3c_id:
1113                 step = "Removing helm-chart-v3 '{}'".format(k8s_hc_id)
1114                 uninstall_sw = deep_get(db_k8scluster, ("_admin", "helm-chart-v3", "created")) or False
1115                 cluster_removed = await self.helm3_k8scluster.reset(cluster_uuid=k8s_h3c_id, uninstall_sw=uninstall_sw)
1116                 db_k8scluster_update["_admin.helm-chart-v3.id"] = None
1117                 db_k8scluster_update["_admin.helm-chart-v3.operationalState"] = "DISABLED"
1118
1119             # Try to remove from cluster_inserted to clean old versions
1120             if k8s_hc_id and cluster_removed:
1121                 step = "Removing k8scluster='{}' from k8srepos".format(k8scluster_id)
1122                 self.logger.debug(logging_text + step)
1123                 db_k8srepo_list = self.db.get_list("k8srepos", {"_admin.cluster-inserted": k8s_hc_id})
1124                 for k8srepo in db_k8srepo_list:
1125                     try:
1126                         cluster_list = k8srepo["_admin"]["cluster-inserted"]
1127                         cluster_list.remove(k8s_hc_id)
1128                         self.update_db_2("k8srepos", k8srepo["_id"], {"_admin.cluster-inserted": cluster_list})
1129                     except Exception as e:
1130                         self.logger.error("{}: {}".format(step, e))
1131             self.db.del_one("k8sclusters", {"_id": k8scluster_id})
1132             db_k8scluster_update = None
1133             self.logger.debug(logging_text + "Done")
1134
1135         except Exception as e:
1136             if isinstance(e, (LcmException, DbException, K8sException, N2VCException, asyncio.CancelledError)):
1137                 self.logger.error(logging_text + "Exit Exception {}".format(e))
1138             else:
1139                 self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1140             exc = e
1141         finally:
1142             if exc and db_k8scluster:
1143                 db_k8scluster_update["_admin.operationalState"] = "ERROR"
1144                 db_k8scluster_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
1145                 # Mark the WIM 'create' HA task as erroneous
1146                 operation_state = 'FAILED'
1147                 operation_details = "ERROR {}: {}".format(step, exc)
1148             else:
1149                 operation_state = 'COMPLETED'
1150                 operation_details = "deleted"
1151
1152             try:
1153                 if db_k8scluster_update:
1154                     self.update_db_2("k8sclusters", k8scluster_id, db_k8scluster_update)
1155                 # Register the K8scluster 'delete' HA task either
1156                 # succesful or erroneous, or do nothing (if legacy NBI)
1157                 self.lcm_tasks.unlock_HA('k8scluster', 'delete', op_id,
1158                                          operationState=operation_state,
1159                                          detailed_status=operation_details)
1160             except DbException as e:
1161                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1162             self.lcm_tasks.remove("k8scluster", k8scluster_id, order_id)
1163
1164
1165 class VcaLcm(LcmBase):
1166     timeout_create = 30
1167
1168     def __init__(self, msg, lcm_tasks, config, loop):
1169         """
1170         Init, Connect to database, filesystem storage, and messaging
1171         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
1172         :return: None
1173         """
1174
1175         self.logger = logging.getLogger("lcm.vca")
1176         self.loop = loop
1177         self.lcm_tasks = lcm_tasks
1178
1179         super().__init__(msg, self.logger)
1180
1181         # create N2VC connector
1182         self.n2vc = N2VCJujuConnector(
1183             log=self.logger,
1184             loop=self.loop,
1185             fs=self.fs,
1186             db=self.db
1187         )
1188
1189     def _get_vca_by_id(self, vca_id: str) -> dict:
1190         db_vca = self.db.get_one("vca", {"_id": vca_id})
1191         self.db.encrypt_decrypt_fields(
1192             db_vca,
1193             "decrypt",
1194             ["secret", "cacert"],
1195             schema_version=db_vca["schema_version"], salt=db_vca["_id"]
1196         )
1197         return db_vca
1198
1199     async def create(self, vca_content, order_id):
1200         op_id = vca_content.pop("op_id", None)
1201         if not self.lcm_tasks.lock_HA("vca", "create", op_id):
1202             return
1203
1204         vca_id = vca_content["_id"]
1205         self.logger.debug("Task vca_create={} {}".format(vca_id, "Enter"))
1206
1207         db_vca = None
1208         db_vca_update = {}
1209
1210         try:
1211             self.logger.debug("Task vca_create={} {}".format(vca_id, "Getting vca from db"))
1212             db_vca = self._get_vca_by_id(vca_id)
1213
1214             task = asyncio.ensure_future(
1215                 asyncio.wait_for(
1216                     self.n2vc.validate_vca(db_vca["_id"]),
1217                     timeout=self.timeout_create,
1218                 )
1219             )
1220
1221             await asyncio.wait([task], return_when=asyncio.FIRST_COMPLETED)
1222             if task.exception():
1223                 raise task.exception()
1224             self.logger.debug("Task vca_create={} {}".format(vca_id, "vca registered and validated successfully"))
1225             db_vca_update["_admin.operationalState"] = "ENABLED"
1226             db_vca_update["_admin.detailed-status"] = "Connectivity: ok"
1227             operation_details = "VCA validated"
1228             operation_state = "COMPLETED"
1229
1230             self.logger.debug("Task vca_create={} {}".format(vca_id, "Done. Result: {}".format(operation_state)))
1231
1232         except Exception as e:
1233             error_msg = "Failed with exception: {}".format(e)
1234             self.logger.error("Task vca_create={} {}".format(vca_id, error_msg))
1235             db_vca_update["_admin.operationalState"] = "ERROR"
1236             db_vca_update["_admin.detailed-status"] = error_msg
1237             operation_state = "FAILED"
1238             operation_details = error_msg
1239         finally:
1240             try:
1241                 self.update_db_2("vca", vca_id, db_vca_update)
1242
1243                 # Register the operation and unlock
1244                 self.lcm_tasks.unlock_HA(
1245                     "vca",
1246                     "create",
1247                     op_id,
1248                     operationState=operation_state,
1249                     detailed_status=operation_details
1250                 )
1251             except DbException as e:
1252                 self.logger.error("Task vca_create={} {}".format(vca_id, "Cannot update database: {}".format(e)))
1253             self.lcm_tasks.remove("vca", vca_id, order_id)
1254
1255     async def delete(self, vca_content, order_id):
1256
1257         # HA tasks and backward compatibility:
1258         # If "vim_content" does not include "op_id", we a running a legacy NBI version.
1259         # In such a case, HA is not supported by NBI, "op_id" is None, and lock_HA() will do nothing.
1260         # Register "delete" task here for related future HA operations
1261         op_id = vca_content.pop("op_id", None)
1262         if not self.lcm_tasks.lock_HA("vca", "delete", op_id):
1263             return
1264
1265         db_vca_update = {}
1266         vca_id = vca_content["_id"]
1267
1268         try:
1269             self.logger.debug("Task vca_delete={} {}".format(vca_id, "Deleting vca from db"))
1270             self.db.del_one("vca", {"_id": vca_id})
1271             db_vca_update = None
1272             operation_details = "deleted"
1273             operation_state = "COMPLETED"
1274
1275             self.logger.debug("Task vca_delete={} {}".format(vca_id, "Done. Result: {}".format(operation_state)))
1276         except Exception as e:
1277             error_msg = "Failed with exception: {}".format(e)
1278             self.logger.error("Task vca_delete={} {}".format(vca_id, error_msg))
1279             db_vca_update["_admin.operationalState"] = "ERROR"
1280             db_vca_update["_admin.detailed-status"] = error_msg
1281             operation_state = "FAILED"
1282             operation_details = error_msg
1283         finally:
1284             try:
1285                 self.update_db_2("vca", vca_id, db_vca_update)
1286                 self.lcm_tasks.unlock_HA(
1287                     "vca",
1288                     "delete",
1289                     op_id,
1290                     operationState=operation_state,
1291                     detailed_status=operation_details,
1292                 )
1293             except DbException as e:
1294                 self.logger.error("Task vca_delete={} {}".format(vca_id, "Cannot update database: {}".format(e)))
1295             self.lcm_tasks.remove("vca", vca_id, order_id)
1296
1297
1298 class K8sRepoLcm(LcmBase):
1299
1300     def __init__(self, msg, lcm_tasks, config, loop):
1301         """
1302         Init, Connect to database, filesystem storage, and messaging
1303         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
1304         :return: None
1305         """
1306
1307         self.logger = logging.getLogger('lcm.k8srepo')
1308         self.loop = loop
1309         self.lcm_tasks = lcm_tasks
1310         self.vca_config = config["VCA"]
1311
1312         super().__init__(msg, self.logger)
1313
1314         self.k8srepo = K8sHelmConnector(
1315             kubectl_command=self.vca_config.get("kubectlpath"),
1316             helm_command=self.vca_config.get("helmpath"),
1317             fs=self.fs,
1318             log=self.logger,
1319             db=self.db,
1320             on_update_db=None
1321         )
1322
1323     async def create(self, k8srepo_content, order_id):
1324
1325         # HA tasks and backward compatibility:
1326         # If 'vim_content' does not include 'op_id', we a running a legacy NBI version.
1327         # In such a case, HA is not supported by NBI, 'op_id' is None, and lock_HA() will do nothing.
1328         # Register 'create' task here for related future HA operations
1329
1330         op_id = k8srepo_content.pop('op_id', None)
1331         if not self.lcm_tasks.lock_HA('k8srepo', 'create', op_id):
1332             return
1333
1334         k8srepo_id = k8srepo_content.get("_id")
1335         logging_text = "Task k8srepo_create={} ".format(k8srepo_id)
1336         self.logger.debug(logging_text + "Enter")
1337
1338         db_k8srepo = None
1339         db_k8srepo_update = {}
1340         exc = None
1341         operation_state = 'COMPLETED'
1342         operation_details = ''
1343         try:
1344             step = "Getting k8srepo-id='{}' from db".format(k8srepo_id)
1345             self.logger.debug(logging_text + step)
1346             db_k8srepo = self.db.get_one("k8srepos", {"_id": k8srepo_id})
1347             db_k8srepo_update["_admin.operationalState"] = "ENABLED"
1348         except Exception as e:
1349             self.logger.error(logging_text + "Exit Exception {}".format(e),
1350                               exc_info=not isinstance(e, (LcmException, DbException, K8sException, N2VCException,
1351                                                           asyncio.CancelledError)))
1352             exc = e
1353         finally:
1354             if exc and db_k8srepo:
1355                 db_k8srepo_update["_admin.operationalState"] = "ERROR"
1356                 db_k8srepo_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
1357                 # Mark the WIM 'create' HA task as erroneous
1358                 operation_state = 'FAILED'
1359                 operation_details = "ERROR {}: {}".format(step, exc)
1360             try:
1361                 if db_k8srepo_update:
1362                     self.update_db_2("k8srepos", k8srepo_id, db_k8srepo_update)
1363                 # Register the K8srepo 'create' HA task either
1364                 # succesful or erroneous, or do nothing (if legacy NBI)
1365                 self.lcm_tasks.unlock_HA('k8srepo', 'create', op_id,
1366                                          operationState=operation_state,
1367                                          detailed_status=operation_details)
1368             except DbException as e:
1369                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1370             self.lcm_tasks.remove("k8srepo", k8srepo_id, order_id)
1371
1372     async def delete(self, k8srepo_content, order_id):
1373
1374         # HA tasks and backward compatibility:
1375         # If 'vim_content' does not include 'op_id', we a running a legacy NBI version.
1376         # In such a case, HA is not supported by NBI, 'op_id' is None, and lock_HA() will do nothing.
1377         # Register 'delete' task here for related future HA operations
1378         op_id = k8srepo_content.pop('op_id', None)
1379         if not self.lcm_tasks.lock_HA('k8srepo', 'delete', op_id):
1380             return
1381
1382         k8srepo_id = k8srepo_content.get("_id")
1383         logging_text = "Task k8srepo_delete={} ".format(k8srepo_id)
1384         self.logger.debug(logging_text + "Enter")
1385
1386         db_k8srepo = None
1387         db_k8srepo_update = {}
1388
1389         exc = None
1390         operation_state = 'COMPLETED'
1391         operation_details = ''
1392         try:
1393             step = "Getting k8srepo-id='{}' from db".format(k8srepo_id)
1394             self.logger.debug(logging_text + step)
1395             db_k8srepo = self.db.get_one("k8srepos", {"_id": k8srepo_id})
1396
1397         except Exception as e:
1398             self.logger.error(logging_text + "Exit Exception {}".format(e),
1399                               exc_info=not isinstance(e, (LcmException, DbException, K8sException, N2VCException,
1400                                                           asyncio.CancelledError)))
1401             exc = e
1402         finally:
1403             if exc and db_k8srepo:
1404                 db_k8srepo_update["_admin.operationalState"] = "ERROR"
1405                 db_k8srepo_update["_admin.detailed-status"] = "ERROR {}: {}".format(step, exc)
1406                 # Mark the WIM 'create' HA task as erroneous
1407                 operation_state = 'FAILED'
1408                 operation_details = "ERROR {}: {}".format(step, exc)
1409             try:
1410                 if db_k8srepo_update:
1411                     self.update_db_2("k8srepos", k8srepo_id, db_k8srepo_update)
1412                 # Register the K8srepo 'delete' HA task either
1413                 # succesful or erroneous, or do nothing (if legacy NBI)
1414                 self.lcm_tasks.unlock_HA('k8srepo', 'delete', op_id,
1415                                          operationState=operation_state,
1416                                          detailed_status=operation_details)
1417                 self.db.del_one("k8srepos", {"_id": k8srepo_id})
1418             except DbException as e:
1419                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1420             self.lcm_tasks.remove("k8srepo", k8srepo_id, order_id)