Addition of PaaS 21/12621/2
authorPatricia Reinoso <patricia.reinoso@canonical.com>
Wed, 26 Oct 2022 08:58:39 +0000 (08:58 +0000)
committerPatricia Reinoso <patricia.reinoso@canonical.com>
Wed, 26 Oct 2022 09:02:49 +0000 (09:02 +0000)
Change-Id: Ieb685b48aba87585f0d5b8bd962265cee5d486ea
Signed-off-by: Patricia Reinoso <patricia.reinoso@canonical.com>
osm_lcm/lcm.py
osm_lcm/lcm_utils.py
osm_lcm/paas.py [new file with mode: 0644]
osm_lcm/tests/test_lcm.py
osm_lcm/tests/test_paas.py [new file with mode: 0644]

index 273edc1..5bbeade 100644 (file)
@@ -29,7 +29,7 @@ import logging.handlers
 import getopt
 import sys
 
-from osm_lcm import ns, vim_sdn, netslice
+from osm_lcm import ns, paas, vim_sdn, netslice
 from osm_lcm.ng_ro import NgRoException, NgRoClient
 from osm_lcm.ROclient import ROClient, ROClientException
 
@@ -121,7 +121,9 @@ class Lcm:
             self.netslice
         ) = (
             self.vim
-        ) = self.wim = self.sdn = self.k8scluster = self.vca = self.k8srepo = None
+        ) = (
+            self.wim
+        ) = self.sdn = self.k8scluster = self.vca = self.k8srepo = self.paas = None
 
         # logging
         log_format_simple = (
@@ -315,6 +317,23 @@ class Lcm:
                 wait_time = 2 if not first_start else 5
                 await asyncio.sleep(wait_time, loop=self.loop)
 
+    def _kafka_read_paas(self, command, params, order_id):
+        paas_id = params.get("_id")
+
+        if command == "created":
+            task = asyncio.ensure_future(self.paas.create(params, order_id))
+            self.lcm_tasks.register("paas", paas_id, order_id, "paas_create", task)
+        elif command == "edited":
+            task = asyncio.ensure_future(self.paas.edit(params, order_id))
+            self.lcm_tasks.register("paas", paas_id, order_id, "paas_edit", task)
+        elif command == "delete":
+            task = asyncio.ensure_future(self.paas.delete(params, order_id))
+            self.lcm_tasks.register("paas", paas_id, order_id, "paas_delete", task)
+        elif command == "deleted":
+            self.logger.debug("PaaS {} already deleted from DB".format(paas_id))
+        else:
+            self.logger.error("Invalid command {} for PaaS topic".format(command))
+
     def kafka_read_callback(self, topic, command, params):
         order_id = 1
 
@@ -383,6 +402,9 @@ class Lcm:
                 task = asyncio.ensure_future(self.vca.delete(params, order_id))
                 self.lcm_tasks.register("vca", vca_id, order_id, "vca_delete", task)
                 return
+        elif topic == "paas":
+            self._kafka_read_paas(command, params, order_id)
+            return
         elif topic == "k8srepo":
             if command == "create" or command == "created":
                 k8srepo_id = params.get("_id")
@@ -672,6 +694,7 @@ class Lcm:
                     "nsi",
                     "k8scluster",
                     "vca",
+                    "paas",
                     "k8srepo",
                     "pla",
                 )
@@ -727,6 +750,7 @@ class Lcm:
             self.msg, self.lcm_tasks, self.config, self.loop
         )
         self.vca = vim_sdn.VcaLcm(self.msg, self.lcm_tasks, self.config, self.loop)
+        self.paas = paas.PaasLcm(self.msg, self.lcm_tasks, self.config, self.loop)
         self.k8srepo = vim_sdn.K8sRepoLcm(
             self.msg, self.lcm_tasks, self.config, self.loop
         )
index 749f347..19852d0 100644 (file)
@@ -379,9 +379,9 @@ class TaskRegistry(LcmBase):
     - worker:  the worker ID for this process
     """
 
-    # NS/NSI: "services" VIM/WIM/SDN: "accounts"
+    # NS/NSI: "services" VIM/WIM/SDN/k8scluster/vca/PaaS/k8srepo: "accounts"
     topic_service_list = ["ns", "nsi"]
-    topic_account_list = ["vim", "wim", "sdn", "k8scluster", "vca", "k8srepo"]
+    topic_account_list = ["vim", "wim", "sdn", "k8scluster", "vca", "paas", "k8srepo"]
 
     # Map topic to InstanceID
     topic2instid_dict = {"ns": "nsInstanceId", "nsi": "netsliceInstanceId"}
@@ -395,6 +395,7 @@ class TaskRegistry(LcmBase):
         "sdn": "sdns",
         "k8scluster": "k8sclusters",
         "vca": "vca",
+        "paas": "paas",
         "k8srepo": "k8srepos",
     }
 
@@ -407,6 +408,7 @@ class TaskRegistry(LcmBase):
             "sdn": {},
             "k8scluster": {},
             "vca": {},
+            "paas": {},
             "k8srepo": {},
         }
         self.worker_id = worker_id
@@ -416,7 +418,7 @@ class TaskRegistry(LcmBase):
     def register(self, topic, _id, op_id, task_name, task):
         """
         Register a new task
-        :param topic: Can be "ns", "nsi", "vim_account", "sdn"
+        :param topic: Can be "ns", "nsi", "vim_account", "sdn", "paas"
         :param _id: _id of the related item
         :param op_id: id of the operation of the related item
         :param task_name: Task descriptive name, as create, instantiate, terminate. Must be unique in this op_id
@@ -588,21 +590,21 @@ class TaskRegistry(LcmBase):
         """
         Lock a task, if possible, to indicate to the HA system that
         the task will be executed in this LCM instance.
-        :param topic: Can be "ns", "nsi", "vim", "wim", or "sdn"
+        :param topic: Can be "ns", "nsi", "vim", "wim", "paas" or "sdn"
         :param op_type: Operation type, can be "nslcmops", "nsilcmops", "create", "edit", "delete"
-        :param op_id: NS, NSI: Operation ID  VIM,WIM,SDN: Account ID + ':' + Operation Index
+        :param op_id: NS, NSI: Operation ID  VIM,WIM,SDN,PaaS: Account ID + ':' + Operation Index
         :return:
         True=lock was successful => execute the task (not registered by any other LCM instance)
         False=lock failed => do NOT execute the task (already registered by another LCM instance)
 
         HA tasks and backward compatibility:
-        If topic is "account type" (VIM/WIM/SDN) and op_id is None, 'op_id' was not provided by NBI.
+        If topic is "account type" (VIM/WIM/SDN/PaaS) and op_id is None, 'op_id' was not provided by NBI.
         This means that the running NBI instance does not support HA.
         In such a case this method should always return True, to always execute
         the task in this instance of LCM, without querying the DB.
         """
 
-        # Backward compatibility for VIM/WIM/SDN/k8scluster without op_id
+        # Backward compatibility for VIM/WIM/SDN/k8scluster/PaaS without op_id
         if self._is_account_type_HA(topic) and op_id is None:
             return True
 
diff --git a/osm_lcm/paas.py b/osm_lcm/paas.py
new file mode 100644 (file)
index 0000000..9c3c0bf
--- /dev/null
@@ -0,0 +1,205 @@
+#!/usr/bin/python3
+
+# Copyright 2022 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+import logging
+from osm_lcm.lcm_utils import LcmBase
+from osm_common.dbbase import DbException
+
+
+class PaasLcm(LcmBase):
+    timeout_create = 30
+
+    def __init__(self, msg, lcm_tasks, config, loop):
+        """Init, Connect to database, filesystem storage, and messaging
+        Args:
+            msg: Message object to be used to write the messages to Kafka Bus
+            lcm_tasks: Task object to register the tasks
+            config: two level dictionary with configuration. Top level should contain 'database', 'storage'
+            loop: Async event loop object
+        """
+
+        self.logger = logging.getLogger("lcm.paas")
+        self.loop = loop
+        self.lcm_tasks = lcm_tasks
+
+        super().__init__(msg, self.logger)
+
+    def _get_paas_by_id(self, paas_id: str) -> dict:
+        db_paas = self.db.get_one("paas", {"_id": paas_id})
+        self.db.encrypt_decrypt_fields(
+            db_paas,
+            "decrypt",
+            ["secret"],
+            schema_version=db_paas["schema_version"],
+            salt=db_paas["_id"],
+        )
+        return db_paas
+
+    def _register_op_and_unlock(
+        self, paas_id, db_paas_update, action, op_id, operation
+    ):
+        """
+        Args:
+            paas_id (str): ID of the PaaS to update.
+            db_paas_update (dict): content to update in DB.
+            action (str): 'create', 'edit', or 'delete'.
+            op_id (str): ID of the operation to unlock.
+            operation (dict): contains 'state' and 'details'.
+        """
+        self.update_db_2("paas", paas_id, db_paas_update)
+        self.lcm_tasks.unlock_HA(
+            "paas",
+            action,
+            op_id,
+            operationState=operation["state"],
+            detailed_status=operation["details"],
+        )
+
+    def _set_status_connectivity_ok(self, db_paas_update: dict, operation: dict):
+        db_paas_update["_admin.operationalState"] = "ENABLED"
+        db_paas_update["_admin.detailed-status"] = "Connectivity: ok"
+        operation["details"] = "PaaS validated"
+        operation["state"] = "COMPLETED"
+
+    def _set_status_exception_raised(
+        self, db_paas_update: dict, operation: dict, error_msg: str
+    ):
+        db_paas_update["_admin.operationalState"] = "ERROR"
+        db_paas_update["_admin.detailed-status"] = error_msg
+        operation["state"] = "FAILED"
+        operation["details"] = error_msg
+
+    async def create(self, paas_content, order_id):
+        """HA tasks and backward compatibility:
+        If "paas_content" does not include "op_id", we a running a legacy NBI version.
+        In such a case, HA is not supported by NBI, "op_id" is None, and lock_HA() will do nothing.
+        Args:
+            paas_content (dict): Contains "op_id" and paas id ("_id")
+            order_id (str): Of the task
+        """
+        op_id = paas_content.pop("op_id", None)
+        if not self.lcm_tasks.lock_HA("paas", "create", op_id):
+            return
+
+        paas_id = paas_content["_id"]
+        self.logger.debug("Task paas_create={} {}".format(paas_id, "Enter"))
+        db_paas_update = {}
+        operation = {}
+
+        try:
+            self._get_paas_by_id(paas_id)
+            self._set_status_connectivity_ok(db_paas_update, operation)
+            msg = "Task paas_create={} Done. Result: {}".format(
+                paas_id, operation["state"]
+            )
+            self.logger.debug(msg)
+
+        except Exception as e:
+            error_msg = "Failed with exception: {}".format(e)
+            self._set_status_exception_raised(db_paas_update, operation, error_msg)
+            self.logger.error("Task paas_create={} {}".format(paas_id, error_msg))
+        finally:
+            try:
+                self._register_op_and_unlock(
+                    paas_id, db_paas_update, "create", op_id, operation
+                )
+            except DbException as e:
+                msg = "Task paas_create={} Cannot update database:{}".format(paas_id, e)
+                self.logger.error(msg)
+            self.lcm_tasks.remove("paas", paas_id, order_id)
+
+    async def edit(self, paas_content, order_id):
+        """HA tasks and backward compatibility:
+        If "paas_content" does not include "op_id", we a running a legacy NBI version.
+        In such a case, HA is not supported by NBI, "op_id" is None, and lock_HA() will do nothing.
+        Args:
+            paas_content (dict): Contains "op_id" and paas id ("_id")
+            order_id (str): Of the task
+        """
+
+        op_id = paas_content.pop("op_id", None)
+        if not self.lcm_tasks.lock_HA("paas", "edit", op_id):
+            return
+
+        paas_id = paas_content["_id"]
+        self.logger.debug("Task paas_edit={} {}".format(paas_id, "Enter"))
+        db_paas_update = {}
+        operation = {}
+
+        try:
+            self._get_paas_by_id(paas_id)
+            self._set_status_connectivity_ok(db_paas_update, operation)
+            msg = "Task paas_edit={} Done. Result: {}".format(
+                paas_id, operation["state"]
+            )
+            self.logger.debug(msg)
+
+        except Exception as e:
+            error_msg = "Failed with exception: {}".format(e)
+            self._set_status_exception_raised(db_paas_update, operation, error_msg)
+            self.logger.error("Task paas_edit={} {}".format(paas_id, error_msg))
+        finally:
+            try:
+                self._register_op_and_unlock(
+                    paas_id, db_paas_update, "edit", op_id, operation
+                )
+            except DbException as e:
+                msg = "Task paas_edit={} Cannot update database:{}".format(paas_id, e)
+                self.logger.error(msg)
+            self.lcm_tasks.remove("paas", paas_id, order_id)
+
+    async def delete(self, paas_content, order_id):
+        """HA tasks and backward compatibility:
+        If "paas_content" does not include "op_id", we a running a legacy NBI version.
+        In such a case, HA is not supported by NBI, "op_id" is None, and lock_HA() will do nothing.
+        Args:
+            paas_content (dict): Contains "op_id" and paas id ("_id")
+            order_id (str): Of the task
+        """
+        op_id = paas_content.pop("op_id", None)
+        if not self.lcm_tasks.lock_HA("paas", "delete", op_id):
+            return
+
+        paas_id = paas_content["_id"]
+        db_paas_update = {}
+        operation = {}
+
+        try:
+            msg = "Task paas_delete={}: Deleting paas from db".format(paas_id)
+            self.logger.debug(msg)
+            self.db.del_one("paas", {"_id": paas_id})
+            db_paas_update = None
+            operation["state"] = "COMPLETED"
+            operation["details"] = "deleted"
+            msg = "Task paas_delete={}: Done. Result: {}".format(
+                paas_id, operation["state"]
+            )
+            self.logger.debug(msg)
+        except Exception as e:
+            error_msg = "Failed with exception: {}".format(e)
+            self.logger.error("Task paas_delete={} {}".format(paas_id, error_msg))
+            self._set_status_exception_raised(db_paas_update, operation, error_msg)
+        finally:
+            try:
+                self._register_op_and_unlock(
+                    paas_id, db_paas_update, "delete", op_id, operation
+                )
+            except DbException as e:
+                msg = "Task paas_delete={}: Cannot update database:{}".format(
+                    paas_id, e
+                )
+                self.logger.error(msg)
+            self.lcm_tasks.remove("paas", paas_id, order_id)
index bdd9a8d..d1a99ff 100644 (file)
@@ -16,7 +16,7 @@ import os
 import re
 import tempfile
 from unittest import TestCase
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
 
 from osm_lcm.lcm import Lcm
 from osm_lcm.data_utils.database.database import Database
@@ -62,7 +62,7 @@ def check_file_content(health_check_file: str) -> str:
         return contents
 
 
-class TestLcm(TestCase):
+class TestLcmBase(TestCase):
     def setUp(self):
         self.config_file = os.getcwd() + "/osm_lcm/tests/test_lcm_config_file.yaml"
         self.config_file_without_storage_path = tempfile.mkstemp()[1]
@@ -77,6 +77,8 @@ class TestLcm(TestCase):
         self.fs.path = "/"
         self.my_lcm = Lcm(config_file=self.config_file)
 
+
+class TestLcm(TestLcmBase):
     def test_get_health_check_file_from_config_file(self):
         self.assertEqual(self.my_lcm.health_check_file, "/tmp/storage/time_last_ping")
 
@@ -86,11 +88,7 @@ class TestLcm(TestCase):
             Lcm(config_file=self.config_file_without_storage_path)
 
     def test_kafka_admin_topic_ping_command(self):
-        params = {
-            "to": "lcm",
-            "from": "lcm",
-            "worker_id": self.my_lcm.worker_id,
-        }
+        params = {"to": "lcm", "from": "lcm", "worker_id": self.my_lcm.worker_id}
         self.my_lcm.health_check_file = tempfile.mkstemp()[1]
         self.my_lcm.kafka_read_callback("admin", "ping", params)
         pattern = "[0-9]{10}.[0-9]{5,8}"
@@ -99,11 +97,7 @@ class TestLcm(TestCase):
         self.assertTrue(result)
 
     def test_kafka_wrong_topic_ping_command(self):
-        params = {
-            "to": "lcm",
-            "from": "lcm",
-            "worker_id": self.my_lcm.worker_id,
-        }
+        params = {"to": "lcm", "from": "lcm", "worker_id": self.my_lcm.worker_id}
         self.my_lcm.health_check_file = tempfile.mkstemp()[1]
         self.my_lcm.kafka_read_callback("kafka", "ping", params)
         pattern = "[0-9]{10}.[0-9]{5,8}"
@@ -112,14 +106,57 @@ class TestLcm(TestCase):
         self.assertFalse(result)
 
     def test_kafka_admin_topic_ping_command_wrong_worker_id(self):
-        params = {
-            "to": "lcm",
-            "from": "lcm",
-            "worker_id": 5,
-        }
+        params = {"to": "lcm", "from": "lcm", "worker_id": 5}
         self.my_lcm.health_check_file = tempfile.mkstemp()[1]
         self.my_lcm.kafka_read_callback("admin", "ping", params)
         pattern = "[0-9]{10}.[0-9]{5,8}"
         # Health check file is empty.
         result = re.findall(pattern, check_file_content(self.my_lcm.health_check_file))
         self.assertFalse(result)
+
+
+@patch("osm_lcm.lcm.asyncio.ensure_future")
+class TestPaasKafkaRead(TestLcmBase):
+    def setUp(self):
+        super().setUp()
+        self.params = {"_id": "paas_id", "name": "paas_name", "type": "juju"}
+        self.order_id = 2
+        self.my_lcm.paas = Mock()
+        self.my_lcm.lcm_tasks = Mock()
+        self.task = {}
+
+    def test_kafka_read_paas_create(self, ensure_future):
+        ensure_future.return_value = self.task
+        self.my_lcm.kafka_read_callback("paas", "created", self.params)
+        self.my_lcm.lcm_tasks.register.assert_called_with(
+            "paas", "paas_id", self.order_id, "paas_create", self.task
+        )
+        ensure_future.assert_called_once_with(self.my_lcm.paas.create())
+
+    def test_kafka_read_paas_update(self, ensure_future):
+        ensure_future.return_value = self.task
+        self.my_lcm.kafka_read_callback("paas", "edited", self.params)
+        self.my_lcm.lcm_tasks.register.assert_called_with(
+            "paas", "paas_id", self.order_id, "paas_edit", self.task
+        )
+        ensure_future.assert_called_once_with(self.my_lcm.paas.edit())
+
+    def test_kafka_read_paas_delete(self, ensure_future):
+        ensure_future.return_value = self.task
+        self.my_lcm.kafka_read_callback("paas", "delete", self.params)
+        self.my_lcm.lcm_tasks.register.assert_called_with(
+            "paas", "paas_id", self.order_id, "paas_delete", self.task
+        )
+        ensure_future.assert_called_once_with(self.my_lcm.paas.delete())
+
+    def test_kafka_read_paas_delete_force(self, ensure_future):
+        ensure_future.return_value = self.task
+        self.my_lcm.kafka_read_callback("paas", "deleted", self.params)
+        self.my_lcm.lcm_tasks.register.assert_not_called()
+        ensure_future.assert_not_called()
+
+    def test_kafka_read_paas_wrong_command(self, ensure_future):
+        ensure_future.return_value = self.task
+        self.my_lcm.kafka_read_callback("paas", "invalid", self.params)
+        self.my_lcm.lcm_tasks.register.assert_not_called()
+        ensure_future.assert_not_called()
diff --git a/osm_lcm/tests/test_paas.py b/osm_lcm/tests/test_paas.py
new file mode 100644 (file)
index 0000000..10da9a5
--- /dev/null
@@ -0,0 +1,303 @@
+#!/usr/bin/python3
+
+# Copyright 2022 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+import asyncio
+from unittest import TestCase
+from unittest.mock import Mock, patch
+from osm_common import msgbase
+from osm_common.dbbase import DbException
+from osm_lcm.paas import PaasLcm
+
+
+class TestPaasLcm(TestCase):
+    @patch("osm_lcm.lcm_utils.Database")
+    @patch("osm_lcm.lcm_utils.Filesystem")
+    def setUp(self, mock_filesystem, mock_database):
+        self.loop = asyncio.get_event_loop()
+        self.msg = Mock(msgbase.MsgBase())
+        self.lcm_tasks = Mock()
+        self.lcm_tasks.lock_HA.return_value = True
+        self.config = {"database": {"driver": "mongo"}}
+        self.paas_lcm = PaasLcm(self.msg, self.lcm_tasks, self.config, self.loop)
+        self.paas_lcm.db = Mock()
+        self.paas_lcm.fs = Mock()
+        self.paas_lcm.update_db_2 = Mock()
+        self.op_id = "op-id"
+        self.paas_id = "paas-id"
+        self.order_id = "order-id"
+        self.paas_content = {"op_id": self.op_id, "_id": self.paas_id}
+        self.db_paas = {
+            "_id": "_id",
+            "name": "paas-name",
+            "secret": "secret",
+            "schema_version": "1.11",
+        }
+
+    def check_assert_not_called_when_legacy_nbi(self):
+        self.paas_lcm.db.get_one.assert_not_called()
+        self.paas_lcm.db.encrypt_decrypt_fields.assert_not_called()
+        self.paas_lcm.update_db_2.assert_not_called()
+        self.lcm_tasks.unlock_HA.assert_not_called()
+        self.lcm_tasks.remove.assert_not_called()
+
+    def check_db_update_when_successful_connectivity(self):
+        self.paas_lcm.update_db_2.assert_called_with(
+            "paas",
+            self.paas_id,
+            {
+                "_admin.operationalState": "ENABLED",
+                "_admin.detailed-status": "Connectivity: ok",
+            },
+        )
+
+    def check_db_update_when_db_exception(self):
+        self.paas_lcm.update_db_2.assert_called_with(
+            "paas",
+            self.paas_id,
+            {
+                "_admin.operationalState": "ERROR",
+                "_admin.detailed-status": "Failed with exception: database exception failed",
+            },
+        )
+
+    def test_paas_lcm_create_legacy_nbi(self):
+        self.lcm_tasks.lock_HA.return_value = False
+        self.loop.run_until_complete(
+            self.paas_lcm.create(self.paas_content, self.order_id)
+        )
+        self.check_assert_not_called_when_legacy_nbi()
+
+    def test_paas_lcm_create(self):
+        self.paas_lcm.db.get_one.return_value = self.db_paas
+        self.loop.run_until_complete(
+            self.paas_lcm.create(self.paas_content, self.order_id)
+        )
+
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "create", self.op_id)
+        self.paas_lcm.db.encrypt_decrypt_fields.assert_called_with(
+            self.db_paas, "decrypt", ["secret"], schema_version="1.11", salt="_id"
+        )
+        self.check_db_update_when_successful_connectivity()
+        self.lcm_tasks.unlock_HA.assert_called_with(
+            "paas",
+            "create",
+            self.op_id,
+            operationState="COMPLETED",
+            detailed_status="PaaS validated",
+        )
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_create_exception_getting_and_updating_db(self):
+        self.paas_lcm.db.get_one.side_effect = DbException("failed")
+        self.paas_lcm.update_db_2.side_effect = DbException("failed")
+        self.loop.run_until_complete(
+            self.paas_lcm.create(self.paas_content, self.order_id)
+        )
+
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "create", self.op_id)
+        self.paas_lcm.db.encrypt_decrypt_fields.assert_not_called()
+
+        self.check_db_update_when_db_exception()
+        self.lcm_tasks.unlock_HA.assert_not_called()
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_create_exception_updating_db(self):
+
+        self.paas_lcm.db.get_one.return_value = self.db_paas
+        self.paas_lcm.update_db_2.side_effect = DbException("failed")
+        self.loop.run_until_complete(
+            self.paas_lcm.create(self.paas_content, self.order_id)
+        )
+
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "create", self.op_id)
+        self.paas_lcm.db.encrypt_decrypt_fields.assert_called_with(
+            self.db_paas, "decrypt", ["secret"], schema_version="1.11", salt="_id"
+        )
+        self.check_db_update_when_successful_connectivity()
+        self.lcm_tasks.unlock_HA.assert_not_called()
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_create_exception_getting_from_db(self):
+        self.paas_lcm.db.get_one.side_effect = DbException("failed")
+        self.loop.run_until_complete(
+            self.paas_lcm.create(self.paas_content, self.order_id)
+        )
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "create", self.op_id)
+        self.paas_lcm.db.encrypt_decrypt_fields.assert_not_called()
+        self.check_db_update_when_db_exception()
+        self.lcm_tasks.unlock_HA.assert_called_with(
+            "paas",
+            "create",
+            self.op_id,
+            operationState="FAILED",
+            detailed_status="Failed with exception: database exception failed",
+        )
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_edit_legacy_nbi(self):
+        self.lcm_tasks.lock_HA.return_value = False
+        self.loop.run_until_complete(
+            self.paas_lcm.edit(self.paas_content, self.order_id)
+        )
+        self.check_assert_not_called_when_legacy_nbi()
+
+    def test_paas_lcm_edit(self):
+
+        self.paas_lcm.db.get_one.return_value = self.db_paas
+        self.loop.run_until_complete(
+            self.paas_lcm.edit(self.paas_content, self.order_id)
+        )
+
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "edit", self.op_id)
+        self.paas_lcm.db.encrypt_decrypt_fields.assert_called_with(
+            self.db_paas, "decrypt", ["secret"], schema_version="1.11", salt="_id"
+        )
+        self.check_db_update_when_successful_connectivity()
+        self.lcm_tasks.unlock_HA.assert_called_with(
+            "paas",
+            "edit",
+            self.op_id,
+            operationState="COMPLETED",
+            detailed_status="PaaS validated",
+        )
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_edit_exception_getting_and_updating_db(self):
+        self.paas_lcm.db.get_one.side_effect = DbException("failed")
+        self.paas_lcm.update_db_2.side_effect = DbException("failed")
+        self.loop.run_until_complete(
+            self.paas_lcm.edit(self.paas_content, self.order_id)
+        )
+
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "edit", self.op_id)
+        self.paas_lcm.db.encrypt_decrypt_fields.assert_not_called()
+
+        self.check_db_update_when_db_exception()
+        self.lcm_tasks.unlock_HA.assert_not_called()
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_edit_exception_updating_db(self):
+        self.paas_lcm.db.get_one.return_value = self.db_paas
+        self.paas_lcm.update_db_2.side_effect = DbException("failed")
+        self.loop.run_until_complete(
+            self.paas_lcm.edit(self.paas_content, self.order_id)
+        )
+
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "edit", self.op_id)
+        self.paas_lcm.db.encrypt_decrypt_fields.assert_called_with(
+            self.db_paas, "decrypt", ["secret"], schema_version="1.11", salt="_id"
+        )
+        self.check_db_update_when_successful_connectivity()
+        self.lcm_tasks.unlock_HA.assert_not_called()
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_edit_exception_getting_from_db(self):
+        self.paas_lcm.db.get_one.side_effect = DbException("failed")
+        self.loop.run_until_complete(
+            self.paas_lcm.edit(self.paas_content, self.order_id)
+        )
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "edit", self.op_id)
+        self.paas_lcm.db.encrypt_decrypt_fields.assert_not_called()
+        self.check_db_update_when_db_exception()
+        self.lcm_tasks.unlock_HA.assert_called_with(
+            "paas",
+            "edit",
+            self.op_id,
+            operationState="FAILED",
+            detailed_status="Failed with exception: database exception failed",
+        )
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_delete_legacy_nbi(self):
+        self.lcm_tasks.lock_HA.return_value = False
+        self.loop.run_until_complete(
+            self.paas_lcm.delete(self.paas_content, self.order_id)
+        )
+        self.check_assert_not_called_when_legacy_nbi()
+
+    def test_paas_lcm_delete(self):
+        self.loop.run_until_complete(
+            self.paas_lcm.delete(self.paas_content, self.order_id)
+        )
+
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "delete", self.op_id)
+        self.paas_lcm.db.del_one.assert_called_with("paas", {"_id": self.paas_id})
+        self.paas_lcm.update_db_2.assert_called_with("paas", self.paas_id, None)
+        self.lcm_tasks.unlock_HA.assert_called_with(
+            "paas",
+            "delete",
+            self.op_id,
+            operationState="COMPLETED",
+            detailed_status="deleted",
+        )
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_delete_exception_deleting_from_db(self):
+        self.paas_lcm.db.del_one.side_effect = Exception("failed deleting")
+        self.loop.run_until_complete(
+            self.paas_lcm.delete(self.paas_content, self.order_id)
+        )
+
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "delete", self.op_id)
+        self.paas_lcm.db.del_one.assert_called_with("paas", {"_id": self.paas_id})
+        self.paas_lcm.update_db_2.assert_called_with(
+            "paas",
+            self.paas_id,
+            {
+                "_admin.operationalState": "ERROR",
+                "_admin.detailed-status": "Failed with exception: failed deleting",
+            },
+        )
+        self.lcm_tasks.unlock_HA.assert_called_with(
+            "paas",
+            "delete",
+            self.op_id,
+            operationState="FAILED",
+            detailed_status="Failed with exception: failed deleting",
+        )
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_delete_exception_updating_db(self):
+        self.loop.run_until_complete(
+            self.paas_lcm.delete(self.paas_content, self.order_id)
+        )
+
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "delete", self.op_id)
+        self.paas_lcm.db.del_one.assert_called_with("paas", {"_id": self.paas_id})
+        self.paas_lcm.update_db_2.assert_called_with("paas", self.paas_id, None)
+        self.lcm_tasks.unlock_HA.not_called()
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)
+
+    def test_paas_lcm_delete_exception_deleting_and_updating_db(self):
+        self.paas_lcm.db.del_one.side_effect = Exception("failed deleting")
+        self.paas_lcm.update_db_2.side_effect = DbException("failed")
+
+        self.loop.run_until_complete(
+            self.paas_lcm.delete(self.paas_content, self.order_id)
+        )
+
+        self.lcm_tasks.lock_HA.assert_called_with("paas", "delete", self.op_id)
+        self.paas_lcm.db.del_one.assert_called_with("paas", {"_id": self.paas_id})
+        self.paas_lcm.update_db_2.assert_called_with(
+            "paas",
+            self.paas_id,
+            {
+                "_admin.operationalState": "ERROR",
+                "_admin.detailed-status": "Failed with exception: failed deleting",
+            },
+        )
+        self.lcm_tasks.unlock_HA.not_called()
+        self.lcm_tasks.remove.assert_called_with("paas", self.paas_id, self.order_id)