Bug 1042: increase nslcmop action default timeout
[osm/LCM.git] / osm_lcm / ns.py
index 07f5845..489c436 100644 (file)
@@ -49,8 +49,8 @@ class NsLcm(LcmBase):
     timeout_ns_deploy = 2 * 3600   # default global timeout for deployment a ns
     timeout_ns_terminate = 1800   # default global timeout for un deployment a ns
     timeout_charm_delete = 10 * 60
-    timeout_primitive = 10 * 60  # timeout for primitive execution
-    timeout_progress_primitive = 2 * 60  # timeout for some progress in a primitive execution
+    timeout_primitive = 30 * 60  # timeout for primitive execution
+    timeout_progress_primitive = 10 * 60  # timeout for some progress in a primitive execution
 
     SUBOPERATION_STATUS_NOT_FOUND = -1
     SUBOPERATION_STATUS_NEW = -2
@@ -785,10 +785,15 @@ class NsLcm(LcmBase):
 
             # Check for and optionally request placement optimization. Database will be updated if placement activated
             stage[2] = "Waiting for Placement."
-            await self._do_placement(logging_text, db_nslcmop, db_vnfrs)
+            if await self._do_placement(logging_text, db_nslcmop, db_vnfrs):
+                # in case of placement change ns_params[vimAcountId) if not present at any vnfrs
+                for vnfr in db_vnfrs.values():
+                    if ns_params["vimAccountId"] == vnfr["vim-account-id"]:
+                        break
+                else:
+                    ns_params["vimAccountId"] == vnfr["vim-account-id"]
 
             # deploy RO
-
             # get vnfds, instantiate at RO
             for c_vnf in nsd.get("constituent-vnfd", ()):
                 member_vnf_index = c_vnf["member-vnf-index"]
@@ -1163,7 +1168,6 @@ class NsLcm(LcmBase):
                     element_under_configuration = kdu_name
 
             # Get artifact path
-            self.fs.sync()  # Sync from FSMongo
             artifact_path = "{}/{}/charms/{}".format(
                 base_folder["folder"],
                 base_folder["pkg-dir"],
@@ -1488,8 +1492,10 @@ class NsLcm(LcmBase):
         :param logging_text: contains the prefix for logging, with the ns and nslcmop identifiers
         :param db_nslcmop: database content of nslcmop
         :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index.
-        :return: None. Modifies database vnfrs and parameter db_vnfr with the computed 'vim-account-id'
+        :return: True if some modification is done. Modifies database vnfrs and parameter db_vnfr with the
+            computed 'vim-account-id'
         """
+        modified = False
         nslcmop_id = db_nslcmop['_id']
         placement_engine = deep_get(db_nslcmop, ('operationParams', 'placement-engine'))
         if placement_engine == "PLA":
@@ -1511,10 +1517,11 @@ class NsLcm(LcmBase):
                 vnfr = db_vnfrs.get(pla_vnf['member-vnf-index'])
                 if not pla_vnf.get('vimAccountId') or not vnfr:
                     continue
+                modified = True
                 self.db.set_one("vnfrs", {"_id": vnfr["_id"]}, {"vim-account-id": pla_vnf['vimAccountId']})
                 # Modifies db_vnfrs
                 vnfr["vim-account-id"] = pla_vnf['vimAccountId']
-        return
+        return modified
 
     def update_nsrs_with_pla_result(self, params):
         try:
@@ -1540,6 +1547,9 @@ class NsLcm(LcmBase):
         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
         self.logger.debug(logging_text + "Enter")
 
+        # Sync from FSMongo
+        self.fs.sync()
+
         # get all needed from database
 
         # database nsrs record
@@ -1595,8 +1605,9 @@ class NsLcm(LcmBase):
             # read from db: ns
             stage[1] = "Getting nsr={} from db".format(nsr_id)
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
-            # nsd is replicated into ns (no db read)
-            nsd = db_nsr["nsd"]
+            stage[1] = "Getting nsd={} from db".format(db_nsr["nsd-id"])
+            nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
+            db_nsr["nsd"] = nsd
             # nsr_name = db_nsr["name"]   # TODO short-name??
 
             # read from db: vnf's of this ns
@@ -1924,14 +1935,13 @@ class NsLcm(LcmBase):
                     error_detail=error_detail,
                     other_update=db_nsr_update
                 )
-            if db_nslcmop:
-                self._write_op_status(
-                    op_id=nslcmop_id,
-                    stage="",
-                    error_message=error_description_nslcmop,
-                    operation_state=nslcmop_operation_state,
-                    other_update=db_nslcmop_update,
-                )
+            self._write_op_status(
+                op_id=nslcmop_id,
+                stage="",
+                error_message=error_description_nslcmop,
+                operation_state=nslcmop_operation_state,
+                other_update=db_nslcmop_update,
+            )
 
             if nslcmop_operation_state:
                 try:
@@ -1957,13 +1967,14 @@ class NsLcm(LcmBase):
 
             # read nsr record
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
+            nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
 
             # this VCA data
             my_vca = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))[vca_index]
 
             # read all ns-configuration relations
             ns_relations = list()
-            db_ns_relations = deep_get(db_nsr, ('nsd', 'ns-configuration', 'relation'))
+            db_ns_relations = deep_get(nsd, ('ns-configuration', 'relation'))
             if db_ns_relations:
                 for r in db_ns_relations:
                     # check if this VCA is in the relation
@@ -2171,7 +2182,7 @@ class NsLcm(LcmBase):
                         storage = deep_get(db_vnfds.get(vnfd_id), ('_admin', 'storage'))
                         if storage and storage.get('pkg-dir'):  # may be not present if vnfd has not artifacts
                             # path format: /vnfdid/pkkdir/helm-charts|juju-bundles/kdumodel
-                            filename = '{}/{}/{}s/{}'.format(storage["folder"], storage["'pkg-dir"], k8sclustertype,
+                            filename = '{}/{}/{}s/{}'.format(storage["folder"], storage["pkg-dir"], k8sclustertype,
                                                              kdumodel)
                             if self.fs.file_exists(filename, mode='file') or self.fs.file_exists(filename, mode='dir'):
                                 kdumodel = self.fs.path + filename
@@ -2384,9 +2395,9 @@ class NsLcm(LcmBase):
             # _ns_execute_primitive() or RO.create_action() will NOT be executed
             return self.SUBOPERATION_STATUS_SKIP
         else:
-            # c. Reintent executing sub-operation
+            # c. retry executing sub-operation
             # The sub-operation exists, and operationState != 'COMPLETED'
-            # Update operationState = 'PROCESSING' to indicate a reintent.
+            # Update operationState = 'PROCESSING' to indicate a retry.
             operationState = 'PROCESSING'
             detailed_status = 'In progress'
             self._update_suboperation_status(
@@ -2399,7 +2410,7 @@ class NsLcm(LcmBase):
     # Find a sub-operation where all keys in a matching dictionary must match
     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
     def _find_suboperation(self, db_nslcmop, match):
-        if (db_nslcmop and match):
+        if db_nslcmop and match:
             op_list = db_nslcmop.get('_admin', {}).get('operations', [])
             for i, op in enumerate(op_list):
                 if all(op.get(k) == match[k] for k in match):
@@ -2467,11 +2478,11 @@ class NsLcm(LcmBase):
     # Check for 3 different cases:
     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP
-    # c. Reintent: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
+    # c. retry: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
     def _check_or_add_scale_suboperation(self, db_nslcmop, vnf_index, vnf_config_primitive, primitive_params,
                                          operationType, RO_nsr_id=None, RO_scaling_info=None):
         # Find this sub-operation
-        if (RO_nsr_id and RO_scaling_info):
+        if RO_nsr_id and RO_scaling_info:
             operationType = 'SCALE-RO'
             match = {
                 'member_vnf_index': vnf_index,
@@ -2750,6 +2761,7 @@ class NsLcm(LcmBase):
         timeout_ns_terminate = self.timeout_ns_terminate
         db_nsr = None
         db_nslcmop = None
+        operation_params = None
         exc = None
         error_list = []   # annotates all failed error messages
         db_nslcmop_update = {}
@@ -2934,14 +2946,14 @@ class NsLcm(LcmBase):
                     error_detail=error_detail,
                     other_update=db_nsr_update
                 )
-            if db_nslcmop:
-                self._write_op_status(
-                    op_id=nslcmop_id,
-                    stage="",
-                    error_message=error_description_nslcmop,
-                    operation_state=nslcmop_operation_state,
-                    other_update=db_nslcmop_update,
-                )
+            self._write_op_status(
+                op_id=nslcmop_id,
+                stage="",
+                error_message=error_description_nslcmop,
+                operation_state=nslcmop_operation_state,
+                other_update=db_nslcmop_update,
+            )
+            if operation_params:
                 autoremove = operation_params.get("autoremove", False)
             if nslcmop_operation_state:
                 try:
@@ -3330,14 +3342,13 @@ class NsLcm(LcmBase):
                     other_update=db_nsr_update
                 )
 
-            if db_nslcmop:
-                self._write_op_status(
-                    op_id=nslcmop_id,
-                    stage="",
-                    error_message=error_description_nslcmop,
-                    operation_state=nslcmop_operation_state,
-                    other_update=db_nslcmop_update,
-                )
+            self._write_op_status(
+                op_id=nslcmop_id,
+                stage="",
+                error_message=error_description_nslcmop,
+                operation_state=nslcmop_operation_state,
+                other_update=db_nslcmop_update,
+            )
 
             if nslcmop_operation_state:
                 try:
@@ -3533,10 +3544,10 @@ class NsLcm(LcmBase):
                         db_nsr_update["config-status"] = "configuring pre-scaling"
                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
 
-                        # Pre-scale reintent check: Check if this sub-operation has been executed before
+                        # Pre-scale retry check: Check if this sub-operation has been executed before
                         op_index = self._check_or_add_scale_suboperation(
                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'PRE-SCALE')
-                        if (op_index == self.SUBOPERATION_STATUS_SKIP):
+                        if op_index == self.SUBOPERATION_STATUS_SKIP:
                             # Skip sub-operation
                             result = 'COMPLETED'
                             result_detail = 'Done'
@@ -3544,20 +3555,20 @@ class NsLcm(LcmBase):
                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
                                                   vnf_config_primitive, result, result_detail))
                         else:
-                            if (op_index == self.SUBOPERATION_STATUS_NEW):
+                            if op_index == self.SUBOPERATION_STATUS_NEW:
                                 # New sub-operation: Get index of this sub-operation
                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
                                                   format(vnf_config_primitive))
                             else:
-                                # Reintent:  Get registered params for this existing sub-operation
+                                # retry:  Get registered params for this existing sub-operation
                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
                                 vnf_index = op.get('member_vnf_index')
                                 vnf_config_primitive = op.get('primitive')
                                 primitive_params = op.get('primitive_params')
-                                self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation reintent".
+                                self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation retry".
                                                   format(vnf_config_primitive))
-                            # Execute the primitive, either with new (first-time) or registered (reintent) args
+                            # Execute the primitive, either with new (first-time) or registered (retry) args
                             result, result_detail = await self._ns_execute_primitive(
                                 self._look_for_deployed_vca(nsr_deployed["VCA"],
                                                             member_vnf_index=vnf_index,
@@ -3581,26 +3592,26 @@ class NsLcm(LcmBase):
             # if (RO_nsr_id and RO_scaling_info):
             if RO_scaling_info:
                 scale_process = "RO"
-                # Scale RO reintent check: Check if this sub-operation has been executed before
+                # Scale RO retry check: Check if this sub-operation has been executed before
                 op_index = self._check_or_add_scale_suboperation(
                     db_nslcmop, vnf_index, None, None, 'SCALE-RO', RO_nsr_id, RO_scaling_info)
-                if (op_index == self.SUBOPERATION_STATUS_SKIP):
+                if op_index == self.SUBOPERATION_STATUS_SKIP:
                     # Skip sub-operation
                     result = 'COMPLETED'
                     result_detail = 'Done'
                     self.logger.debug(logging_text + "Skipped sub-operation RO, result {} {}".format(
                         result, result_detail))
                 else:
-                    if (op_index == self.SUBOPERATION_STATUS_NEW):
+                    if op_index == self.SUBOPERATION_STATUS_NEW:
                         # New sub-operation: Get index of this sub-operation
                         op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
                         self.logger.debug(logging_text + "New sub-operation RO")
                     else:
-                        # Reintent:  Get registered params for this existing sub-operation
+                        # retry:  Get registered params for this existing sub-operation
                         op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
                         RO_nsr_id = op.get('RO_nsr_id')
                         RO_scaling_info = op.get('RO_scaling_info')
-                        self.logger.debug(logging_text + "Sub-operation RO reintent".format(
+                        self.logger.debug(logging_text + "Sub-operation RO retry for primitive {}".format(
                             vnf_config_primitive))
 
                     RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
@@ -3726,7 +3737,7 @@ class NsLcm(LcmBase):
                         db_nsr_update["config-status"] = "configuring post-scaling"
                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
 
-                        # Post-scale reintent check: Check if this sub-operation has been executed before
+                        # Post-scale retry check: Check if this sub-operation has been executed before
                         op_index = self._check_or_add_scale_suboperation(
                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'POST-SCALE')
                         if op_index == self.SUBOPERATION_STATUS_SKIP:
@@ -3743,14 +3754,14 @@ class NsLcm(LcmBase):
                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
                                                   format(vnf_config_primitive))
                             else:
-                                # Reintent:  Get registered params for this existing sub-operation
+                                # retry:  Get registered params for this existing sub-operation
                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
                                 vnf_index = op.get('member_vnf_index')
                                 vnf_config_primitive = op.get('primitive')
                                 primitive_params = op.get('primitive_params')
-                                self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation reintent".
+                                self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation retry".
                                                   format(vnf_config_primitive))
-                            # Execute the primitive, either with new (first-time) or registered (reintent) args
+                            # Execute the primitive, either with new (first-time) or registered (retry) args
                             result, result_detail = await self._ns_execute_primitive(
                                 self._look_for_deployed_vca(nsr_deployed["VCA"],
                                                             member_vnf_index=vnf_index,
@@ -3769,9 +3780,6 @@ class NsLcm(LcmBase):
                         scale_process = None
             # POST-SCALE END
 
-            db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
-            db_nslcmop_update["statusEnteredTime"] = time()
-            db_nslcmop_update["detailed-status"] = "done"
             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
                 else old_operational_status
@@ -3794,10 +3802,8 @@ class NsLcm(LcmBase):
                 current_operation_id=None
             )
             if exc:
-                if db_nslcmop:
-                    db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
-                    db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
-                    db_nslcmop_update["statusEnteredTime"] = time()
+                db_nslcmop_update["detailed-status"] = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
+                nslcmop_operation_state = "FAILED"
                 if db_nsr:
                     db_nsr_update["operational-status"] = old_operational_status
                     db_nsr_update["config-status"] = old_config_status
@@ -3809,20 +3815,27 @@ class NsLcm(LcmBase):
                             db_nsr_update["operational-status"] = "failed"
                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
                                                                                                      exc)
-            try:
-                if db_nslcmop and db_nslcmop_update:
-                    self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
-                if db_nsr:
-                    self._write_ns_status(
-                        nsr_id=nsr_id,
-                        ns_state=None,
-                        current_operation="IDLE",
-                        current_operation_id=None,
-                        other_update=db_nsr_update
-                    )
+            else:
+                error_description_nslcmop = None
+                nslcmop_operation_state = "COMPLETED"
+                db_nslcmop_update["detailed-status"] = "Done"
+
+            self._write_op_status(
+                op_id=nslcmop_id,
+                stage="",
+                error_message=error_description_nslcmop,
+                operation_state=nslcmop_operation_state,
+                other_update=db_nslcmop_update,
+            )
+            if db_nsr:
+                self._write_ns_status(
+                    nsr_id=nsr_id,
+                    ns_state=None,
+                    current_operation="IDLE",
+                    current_operation_id=None,
+                    other_update=db_nsr_update
+                )
 
-            except DbException as e:
-                self.logger.error(logging_text + "Cannot update database: {}".format(e))
             if nslcmop_operation_state:
                 try:
                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,