Bug 22 : SO is marked as active before proxy charms 78/178/1
authorPhilip Joseph <Philip.Joseph@riftio.com>
Wed, 15 Jun 2016 13:05:36 +0000 (18:35 +0530)
committerjosephp <Philip.Joseph@riftio.com>
Wed, 15 Jun 2016 17:14:51 +0000 (22:44 +0530)
Signed-off-by: josephp <Philip.Joseph@riftio.com>
.gitignore
modules/core/mano/rwcm/plugins/rwconman/rift/tasklets/rwconmantasklet/RiftCA.py
modules/core/mano/rwcm/plugins/rwconman/rift/tasklets/rwconmantasklet/juju_intf.py
modules/core/mano/rwcm/plugins/rwconman/rift/tasklets/rwconmantasklet/jujuconf.py
modules/core/mano/rwcm/plugins/rwconman/rift/tasklets/rwconmantasklet/riftcm_config_plugin.py
modules/core/mano/rwcm/plugins/rwconman/rift/tasklets/rwconmantasklet/rwconman_conagent.py
modules/core/mano/rwcm/plugins/rwconman/rift/tasklets/rwconmantasklet/rwconman_config.py
modules/core/mano/rwcm/plugins/rwconman/rift/tasklets/rwconmantasklet/rwconmantasklet.py

index 791a558..f3986a4 100644 (file)
@@ -11,6 +11,8 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
+
+.artifacts
 .build
 *.tgz
 .install
@@ -34,6 +36,7 @@ scripts/rpm/
 scripts/system
 scripts/test/
 scripts/util/
+scripts/packaging/
 .gitmodules.deps.orig
 .gitmodules.orig
 modules/toolchain/
index ab0181e..fa3398d 100644 (file)
@@ -48,7 +48,7 @@ class RiftCAConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
         return self._name
         
     @asyncio.coroutine
-    def notify_create_vls(self, agent_nsr, agent_vnfr, vld, vlr):
+    def notify_create_vlr(self, agent_nsr, agent_vnfr, vld, vlr):
         """
         Notification of create VL record
         """
@@ -75,7 +75,7 @@ class RiftCAConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
         """
         # Deploy the charm if specified for the vnf
         self._log.debug("Rift config agent: create vnfr nsr={}  vnfr={}"
-                        .format(agent_nsr, agent_vnfr.name))
+                        .format(agent_nsr.name, agent_vnfr.name))
         try:
             self._loop.create_task(self.is_vnf_configurable(agent_vnfr))
         except Exception as e:
@@ -86,27 +86,27 @@ class RiftCAConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
         return True
 
     @asyncio.coroutine
-    def notify_instantiate_vnf(self, agent_nsr, agent_vnfr):
+    def notify_instantiate_vnfr(self, agent_nsr, agent_vnfr):
         """
         Notification of Instantiate NSR with the passed nsr id
         """
         pass
 
     @asyncio.coroutine
-    def notify_instantiate_vl(self, agent_nsr, agent_vnfr, vlr):
+    def notify_instantiate_vlr(self, agent_nsr, agent_vnfr, vlr):
         """
         Notification of Instantiate NSR with the passed nsr id
         """
         pass
 
     @asyncio.coroutine
-    def notify_terminate_vnf(self, agent_nsr, agent_vnfr):
+    def notify_terminate_vnfr(self, agent_nsr, agent_vnfr):
         """
         Notification of Terminate the network service
         """
 
     @asyncio.coroutine
-    def notify_terminate_vl(self, agent_nsr, agent_vnfr, vlr):
+    def notify_terminate_vlr(self, agent_nsr, agent_vnfr, vlr):
         """
         Notification of Terminate the virtual link
         """
@@ -249,7 +249,14 @@ class RiftCAConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
         if agent_vnfr.id not in self._rift_vnfs.keys():
             self._log.info("Rift config agent: add vnfr={}/{}".format(agent_vnfr.name, agent_vnfr.id))
             self._rift_vnfs[agent_vnfr.id] = agent_vnfr
-            
+
+    @asyncio.coroutine
+    def get_config_status(self, agent_nsr, agent_vnfr):
+            if agent_vnfr.id in self._rift_vnfs.keys():
+                return 'configured'
+            return 'unknown'
+
+
     def get_action_status(self, execution_id):
         ''' Get the action status for an execution ID
             *** Make sure this is NOT a asyncio coroutine function ***
index 4732342..dcf2bbc 100644 (file)
@@ -835,13 +835,13 @@ class JujuApi(object):
         return False
 
     def is_service_up(self, service):
-        if self.get_service_status in ['active', 'blocked']:
+        if self.get_service_status(service) in ['active', 'blocked']:
             return True
 
         return False
 
     def is_service_in_error(self, service):
-        if self.get_service_status == 'error':
+        if self.get_service_status(service) == 'error':
             self.log.debug("Juju: service is in error state for %s" % service)
 
     def wait_for_service(self, service):
@@ -1090,4 +1090,4 @@ class JujuApi(object):
                 return
             srv_status = self.get_service_status(service, status)
 
-        self.log.debug("Destroyed service %s (%s)" % (service, srv_status))
\ No newline at end of file
+        self.log.debug("Destroyed service %s (%s)" % (service, srv_status))
index 069b249..c951f12 100644 (file)
@@ -23,10 +23,6 @@ import tempfile
 import yaml
 import os
 
-from gi.repository import (
-    RwDts as rwdts,
-)
-
 from . import juju_intf
 from . import riftcm_config_plugin
 
@@ -200,7 +196,7 @@ class JujuConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
         return val
 
     @asyncio.coroutine
-    def notify_create_vls(self, agent_nsr, agent_vnfr, vld, vlr):
+    def notify_create_vlr(self, agent_nsr, agent_vnfr, vld, vlr):
         """
         Notification of create VL record
         """
@@ -280,28 +276,28 @@ class JujuConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
         return True
 
     @asyncio.coroutine
-    def notify_instantiate_vnf(self, agent_nsr, agent_vnfr):
+    def notify_instantiate_vnfr(self, agent_nsr, agent_vnfr):
         """
         Notification of Instantiate NSR with the passed nsr id
         """
         return True
 
     @asyncio.coroutine
-    def notify_instantiate_vl(self, agent_nsr, agent_vnfr, vlr):
+    def notify_instantiate_vlr(self, agent_nsr, agent_vnfr, vlr):
         """
         Notification of Instantiate NSR with the passed nsr id
         """
         return True
 
     @asyncio.coroutine
-    def notify_terminate_ns(self, agent_nsr, agent_vnfr):
+    def notify_terminate_nsr(self, agent_nsr, agent_vnfr):
         """
         Notification of Terminate the network service
         """
         return True
 
     @asyncio.coroutine
-    def notify_terminate_vnf(self, agent_nsr, agent_vnfr):
+    def notify_terminate_vnfr(self, agent_nsr, agent_vnfr):
         """
         Notification of Terminate the network service
         """
@@ -338,7 +334,7 @@ class JujuConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
         return True
 
     @asyncio.coroutine
-    def notify_terminate_vl(self, agent_nsr, agent_vnfr, vlr):
+    def notify_terminate_vlr(self, agent_nsr, agent_vnfr, vlr):
         """
         Notification of Terminate the virtual link
         """
@@ -583,8 +579,10 @@ class JujuConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
         vnfr = agent_vnfr.vnfr
         service = vnfr['vnf_juju_name']
 
-        if not self.check_task_status(vnfr['vnf_juju_name'], 'deploy'):
+        rc = yield from self.is_service_up(service)
+        if not rc:
             return False
+
         try:
             tags = []
             api = None
@@ -666,7 +664,7 @@ class JujuConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
         if agent_vnfr.id not in self._juju_vnfs.keys():
             self._log.info("juju config agent: add vnfr={}/{}".format(agent_vnfr.name, agent_vnfr.id))
             self._juju_vnfs[agent_vnfr.id] = agent_vnfr
-            
+
     def is_vnfr_managed(self, vnfr_id):
         try:
             if vnfr_id in self._juju_vnfs:
@@ -685,17 +683,15 @@ class JujuConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
                 self._log.error("jujuCA: Unable to get API for checking service is active")
                 return resp
 
-            for vnf in self._juju_vnfs:
-                if vnf['name'] == service and api:
-                    # Check if deploy is over
-                    if self.check_task_status(service, 'deploy'):
-                        resp = yield from self._loop.run_in_executor(
-                            None,
-                            api.is_service_active,
-                            service
-                        )
-                    self._log.debug("jujuCA: Is the service %s active? %s", service, resp)
-                    return resp
+            # Check if deploy is over
+            if self.check_task_status(service, 'deploy'):
+                resp = yield from self._loop.run_in_executor(
+                    None,
+                    api.is_service_active,
+                    service
+                )
+                self._log.debug("jujuCA: Is the service %s active? %s", service, resp)
+                return resp
         except KeyError:
             self._log.error("jujuCA: Check active unknown service ", service)
         except Exception as e:
@@ -703,6 +699,31 @@ class JujuConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
             self._log.exception(e)
         return resp
 
+    def is_service_up(self, service):
+        """ Is the juju service up (active or blocked)  """
+        resp = False
+        try:
+            api = yield from self._get_api()
+            if api is None:
+                self._log.error("jujuCA: Unable to get API for checking service is active")
+                return resp
+
+            # Check if deploy is over
+            if self.check_task_status(service, 'deploy'):
+                resp = yield from self._loop.run_in_executor(
+                    None,
+                    api.is_service_up,
+                    service
+                )
+            self._log.debug("jujuCA: Is the service %s up? %s", service, resp)
+            return resp
+        except KeyError:
+            self._log.error("jujuCA: Check unknown service %s is up", service)
+        except Exception as e:
+            self._log.error("jujuCA: Caught exception when checking for service is active: %s", e)
+            self._log.exception(e)
+        return resp
+
     @asyncio.coroutine
     def is_configured(self, vnfr_id):
         try:
@@ -730,44 +751,46 @@ class JujuConfigPlugin(riftcm_config_plugin.RiftCMConfigPluginBase):
         return False
 
     @asyncio.coroutine
-    def get_status(self, vnfr_id):
-        resp = 'unknown'
-        try:
-            vnfr = self._juju_vnfs[vnfr_id]
-            if vnfr['active']:
-                return 'configured'
+    def get_config_status(self, agent_nsr, agent_vnfr):
+        """Get the configuration status for the VNF"""
+        rc = 'unknown'
 
+        try:
+            vnfr = agent_vnfr.vnfr
             service = vnfr['vnf_juju_name']
-            # Check if deploy is over
-            if self.check_task_status(service, 'deploy'):
-                api = yield from self._get_api()
-                if api is None:
-                    self._log.error("jujuCA: API not created for get status")
-                    return 'failed'
+        except KeyError:
+            # This VNF is not managed by Juju
+            return rc
 
-                resp = yield from self._loop.run_in_executor(
-                    None,
-                    api.get_service_status,
-                    service
-                )
-            self._log.debug("jujuCA: Service status for {} is {}".
-                            format(service, resp))
-            status =  'configuring'
-            if resp in ['active', 'blocked']:
-                vnfr['active'] = True
-                status = 'configured'
-            elif resp in ['error', 'NA']:
-                status = 'failed'
-            return status
-        except KeyError as e:
-            self._log.debug("jujuCA: VNFR id {} not found in config agent, e={}".
-                            format(vnfr_id, e))
-            return 'configured'
+        rc = 'configuring'
+
+        if not self.check_task_status(service, 'deploy'):
+            return rc
+
+        try:
+            api = yield from self._get_api()
+            if api is None:
+                self._log.error("jujuCA: Unable to get API for checking service is active")
+                return rc
+
+            resp = yield from self._loop.run_in_executor(
+                None,
+                api.get_service_status,
+                service
+            )
+            self._log.debug("jujuCA: Get service %s status? %s", service, resp)
+
+            if resp == 'error':
+                return 'error'
+            if resp == 'active':
+                return 'configured'
+        except KeyError:
+            self._log.error("jujuCA: Check unknown service %s status", service)
         except Exception as e:
-            self._log.error("jujuCA: VNFR id {} gt_status, e={}".
-                            format(vnfr_id, e))
+            self._log.error("jujuCA: Caught exception when checking for service is active: %s", e)
             self._log.exception(e)
-        return resp
+
+        return rc
 
     def get_action_status(self, execution_id):
         ''' Get the action status for an execution ID
index 995180f..93086c9 100644 (file)
@@ -170,7 +170,7 @@ class RiftCMConfigPluginBase(object):
 
     @abc.abstractmethod
     @asyncio.coroutine
-    def notify_create_vls(self, agent_nsr, vld):
+    def notify_create_vlr(self, agent_nsr, vld):
         """ Notification on creation of an VL """
         pass
 
@@ -182,25 +182,25 @@ class RiftCMConfigPluginBase(object):
 
     @abc.abstractmethod
     @asyncio.coroutine
-    def notify_instantiate_vnf(self, agent_nsr, agent_vnfr):
+    def notify_instantiate_vnfr(self, agent_nsr, agent_vnfr):
         """ Notify instantiation of the virtual network function """
         pass
 
     @abc.abstractmethod
     @asyncio.coroutine
-    def notify_instantiate_vl(self, agent_nsr, vl):
+    def notify_instantiate_vlr(self, agent_nsr, vl):
         """ Notify instantiate of the virtual link"""
         pass
 
     @abc.abstractmethod
     @asyncio.coroutine
-    def notify_terminate_vnf(self, agent_nsr, agent_vnfr):
+    def notify_terminate_vnfr(self, agent_nsr, agent_vnfr):
         """Notify termination of the VNF """
         pass
 
     @abc.abstractmethod
     @asyncio.coroutine
-    def notify_terminate_vl(self, agent_nsr, vlr):
+    def notify_terminate_vlr(self, agent_nsr, vlr):
         """Notify termination of the Virtual Link Record"""
         pass
 
@@ -240,35 +240,37 @@ class RiftCMConfigPluginBase(object):
     @asyncio.coroutine
     def invoke(self, method, *args):
         try:
-            rc = False
+            rc = None
             self._log.debug("Config agent plugin: method {} with args {}: {}".
                             format(method, args, self))
-            
+
             # TBD - Do a better way than string compare to find invoke the method
             if method == 'notify_create_nsr':
                 rc = yield from self.notify_create_nsr(args[0], args[1])
-            elif method == 'notify_create_vls':
-                rc = yield from self.notify_create_vls(args[0], args[1], args[2])
+            elif method == 'notify_create_vlr':
+                rc = yield from self.notify_create_vlr(args[0], args[1], args[2])
             elif method == 'notify_create_vnfr':
                 rc = yield from self.notify_create_vnfr(args[0], args[1])
-            elif method == 'notify_instantiate_ns':
-                rc = yield from self.notify_instantiate_ns(args[0])
-            elif method == 'notify_instantiate_vnf':
-                rc = yield from self.notify_instantiate_vnf(args[0], args[1])
-            elif method == 'notify_instantiate_vl':
-                rc = yield from self.notify_instantiate_vl(args[0], args[1])
+            elif method == 'notify_instantiate_nsr':
+                rc = yield from self.notify_instantiate_nsr(args[0])
+            elif method == 'notify_instantiate_vnfr':
+                rc = yield from self.notify_instantiate_vnfr(args[0], args[1])
+            elif method == 'notify_instantiate_vlr':
+                rc = yield from self.notify_instantiate_vlr(args[0], args[1])
             elif method == 'notify_nsr_active':
                 rc = yield from self.notify_nsr_active(args[0], args[1])
-            elif method == 'notify_terminate_ns':
-                rc = yield from self.notify_terminate_ns(args[0])
-            elif method == 'notify_terminate_vnf':
-                rc = yield from self.notify_terminate_vnf(args[0], args[1])
-            elif method == 'notify_terminate_vl':
-                rc = yield from self.notify_terminate_vl(args[0], args[1])
+            elif method == 'notify_terminate_nsr':
+                rc = yield from self.notify_terminate_nsr(args[0])
+            elif method == 'notify_terminate_vnfr':
+                rc = yield from self.notify_terminate_vnfr(args[0], args[1])
+            elif method == 'notify_terminate_vlr':
+                rc = yield from self.notify_terminate_vlr(args[0], args[1])
             elif method == 'apply_initial_config':
                 rc = yield from self.apply_initial_config(args[0], args[1])
             elif method == 'apply_config':
                 rc = yield from self.apply_config(args[0], args[1], args[2])
+            elif method == 'get_config_status':
+                rc = yield from self.get_config_status(args[0], args[1])
             else:
                 self._log.error("Unknown method %s invoked on config agent plugin",
                                 method)
index c308eaf..0bccdaf 100644 (file)
@@ -27,10 +27,22 @@ from . import jujuconf
 import rift.mano.config_agent
 
 
+class ConfigAgentError(Exception):
+    pass
+
+
+class ConfigAgentExistsError(ConfigAgentError):
+    pass
+
 
-class ConfigAgentExistsError(Exception):
+class UnknownAgentTypeError(Exception):
     pass
 
+
+class ConfigAgentVnfrAddError(Exception):
+    pass
+
+
 class ConfigAccountHandler(object):
     def __init__(self, dts, log, loop, on_add_config_agent):
         self._log = log
@@ -155,25 +167,27 @@ class RiftCMConfigAgent(object):
             cap_inst = self._config_plugins.class_by_plugin_name(
                 config_agent.account_type)
         except KeyError as e:
-            self._log.debug(
-                "Config agent nsm plugin type not found: {}.  Using default plugin, e={}".
-                format(config_agent.account_type, e))
-            cap_name = self.DEFAULT_CAP_TYPE
-            cap_inst = self._config_plugins.class_by_plugin_name(cap_name)
+            msg = "Config agent nsm plugin type not found: {}". \
+                format(config_agent.account_type)
+            self._log.error(msg)
+            raise UnknownAgentTypeError(msg)
 
         # Check to see if the plugin was already instantiated
         if cap_name in self._plugin_instances:
-            self._log.debug("Config agent nsm plugin already instantiated.  Using existing.")
+            self._log.debug("Config agent nsm plugin {} already instantiated. " \
+                            "Using existing.". format(cap_name))
         else:
             # Otherwise, instantiate a new plugin using the config agent account
             self._log.debug("Instantiting new config agent using class: %s", cap_inst)
             new_instance = cap_inst(self._dts, self._log, self._loop, config_agent)
             self._plugin_instances[cap_name] = new_instance
 
-        if self._default_account_added:
-            # If the user has provided a config account, chuck the default one.
-            if self.DEFAULT_CAP_TYPE in self._plugin_instances:
-                del self._plugin_instances[self.DEFAULT_CAP_TYPE]
+        # TODO (pjoseph): See why this was added, as this deletes the
+        # Rift plugin account when Juju account is added
+        # if self._default_account_added:
+        #     # If the user has provided a config account, chuck the default one.
+        #     if self.DEFAULT_CAP_TYPE in self._plugin_instances:
+        #         del self._plugin_instances[self.DEFAULT_CAP_TYPE]
 
 
 
@@ -197,13 +211,18 @@ class RiftCMConfigAgent(object):
         if method == 'juju':
             agent_type = 'juju'
         elif method in ['netconf', 'script']:
-            agent_type = 'riftca'
+            agent_type = self.DEFAULT_CAP_TYPE
         else:
-            self._log.error("Unsupported configuration method ({}) for VNF:{}/{}".format(method, nsr.name, vnfr.name))
+            msg = "Unsupported configuration method ({}) for VNF:{}/{}". \
+                  format(method, nsr.name, vnfr.name)
+            self._log.error(msg)
+            raise UnknownAgentTypeError(msg)
 
         try:
-            if agent_type in self._plugin_instances.keys():
-                agent = self._plugin_instances[agent_type]
-                agent.add_vnfr_managed(vnfr)
+            agent = self._plugin_instances[agent_type]
+            agent.add_vnfr_managed(vnfr)
         except Exception as e:
-            self._log.error("Error set_config_agent={}".format(str(e)))
\ No newline at end of file
+            self._log.error("Error set_config_agent for type {}: {}".
+                            format(agent_type, str(e)))
+            self._log.exception(e)
+            raise ConfigAgentVnfrAddError(e)
index 3d0e56f..76a8461 100644 (file)
@@ -156,7 +156,12 @@ class ConfigManagerConfig(object):
                                         PretendNsm(
                                             self._dts, self._log, self._loop, self)),
         ]
-        
+
+    def is_nsr_valid(self, nsr_id):
+        if nsr_id in self._nsr_dict:
+            return True
+        return False
+
     def add_to_pending_tasks(self, task):
         if self.pending_tasks:
             for p_task in self.pending_tasks:
@@ -516,8 +521,9 @@ class ConfigManagerConfig(object):
                             yield from self.process_nsd_vnf_configuration(nsr_obj, vnfr)
                             yield from self._config_agent_mgr.invoke_config_agent_plugins(
                                 'notify_create_vnfr',
-                                nsr_obj.agent_nsr, agent_vnfr)
-                        
+                                nsr_obj.agent_nsr,
+                                agent_vnfr)
+
                         #####################TBD###########################
                         # self._log.debug("VNF active. Apply initial config for vnfr {}".format(vnfr.name))
                         # yield from self._config_agent_mgr.invoke_config_agent_plugins('apply_initial_config',
@@ -625,16 +631,17 @@ class ConfigManagerConfig(object):
             # Call Config Agent to clean up for each VNF
             for agent_vnfr in nsr_obj.agent_nsr.vnfrs:
                 yield from self._config_agent_mgr.invoke_config_agent_plugins(
-                    'notify_terminate_vnf',
-                    nsr_obj.agent_nsr, agent_vnfr)
+                    'notify_terminate_vnfr',
+                    nsr_obj.agent_nsr,
+                    agent_vnfr)
 
             # publish delete cm-state (cm-nsr)
             yield from nsr_obj.delete_cm_nsr()
 
             #####################TBD###########################
             # yield from self._config_agent_mgr.invoke_config_agent_plugins('notify_terminate_ns', self.id)
-            
-            self._log.critical("NSR(%s/%s) is deleted", nsr_obj.nsr_name, id)
+
+            self._log.info("NSR(%s/%s) is deleted", nsr_obj.nsr_name, id)
 
 class ConfigManagerNSR(object):
     def __init__(self, log, loop, parent, id):
@@ -669,6 +676,14 @@ class ConfigManagerNSR(object):
         # Place holders for NSR & VNFR classes
         self.agent_nsr = None
 
+    @property
+    def nsr_id(self):
+        return self._nsr_id
+
+    @property
+    def parent(self):
+        return self._parent
+
     @property
     def nsr_opdata_xpath(self):
         ''' Returns full xpath for this NSR cm-state opdata '''
@@ -977,7 +992,7 @@ class ConfigManagerNSR(object):
             'protocol' : 'None',
             'mgmt_ip_address' : '0.0.0.0',
             'cfg_file' : 'None',
-            'cfg_retries' : 5,
+            'cfg_retries' : 0,
             'script_type' : 'bash',
         }
 
@@ -1185,4 +1200,4 @@ class ConfigManagerDTS(object):
                                                              handler=handler)
         except Exception as e:
             self._log.error("Failed to register for NSR changes as %s", str(e))
-            
\ No newline at end of file
+
index 81edeb3..33c392b 100755 (executable)
@@ -71,6 +71,7 @@ class ConfigurationManager(object):
         self._config         = Config.ConfigManagerConfig(self._dts, self._log, self._loop, self)
         self._event          = Event.ConfigManagerEvents(self._dts, self._log, self._loop, self)
         self.pending_cfg     = []
+        self.pending_tasks   = {}
 
         self._handlers = [
             self._config,
@@ -88,8 +89,6 @@ class ConfigurationManager(object):
     def update_ns_state(self, nsr_obj, state):
         self._log.info("Updating cm-state for NS(%s) to:%s", nsr_obj.nsr_name, state)
         yield from nsr_obj.update_ns_cm_state(state)
-        # Publish cm-state
-        yield from nsr_obj.publish_cm_state()
 
     def add_to_pending(self, nsr_obj):
         
@@ -111,31 +110,48 @@ class ConfigurationManager(object):
                     self._log.debug("Adding VNF:(%s) to pending cfg list", log_this_vnf(vnfr['vnf_cfg']))
                     nsr_obj.vnf_cfg_list.append(vnfr['vnf_cfg'])
             self.pending_cfg.append(nsr_obj)
-            
+
     @asyncio.coroutine
     def configuration_handler(self):
         @asyncio.coroutine
         def process_vnf_cfg(agent_vnfr, nsr_obj):
             vnf_cfg = agent_vnfr.vnf_cfg
             done = False
+
             if vnf_cfg['cfg_retries']:
-                if vnf_cfg['cfg_retries'] < 5:
-                    # This failed previously, lets give it some time
-                    yield from asyncio.sleep(5, loop=self._loop)
-                vnf_cfg['cfg_retries'] -= 1
-                done = yield from self._config._config_agent_mgr.invoke_config_agent_plugins(
-                    'apply_initial_config', nsr_obj.agent_nsr, agent_vnfr)
-                self._log.debug("Apply configuration for VNF={} returned {}".
-                                format(log_this_vnf(vnf_cfg), done))
-                if done:
-                    yield from self.update_vnf_state(vnf_cfg, conmanY.RecordState.READY)
-                elif vnf_cfg['cfg_retries'] == 0:
-                    # Failed configuration after max retries
+                # This failed previously, lets give it some time
+                yield from asyncio.sleep(5, loop=self._loop)
+
+            vnf_cfg['cfg_retries'] += 1
+
+            # Check to see if this vnfr is managed
+            done = yield from self._config._config_agent_mgr.invoke_config_agent_plugins(
+                'apply_initial_config',
+                nsr_obj.agent_nsr,
+                agent_vnfr)
+            self._log.debug("Apply configuration for VNF={} on attempt {} " \
+                            "returned {}".format(log_this_vnf(vnf_cfg),
+                                                 vnf_cfg['cfg_retries'],
+                                                 done))
+
+            if done:
+                yield from self.update_vnf_state(vnf_cfg, conmanY.RecordState.READY)
+
+            else:
+                # Check to see if the VNF configure failed
+                status = yield from self._config._config_agent_mgr.invoke_config_agent_plugins(
+                    'get_config_status',
+                    nsr_obj.agent_nsr,
+                    agent_vnfr)
+
+                if status and status == 'failed':
+                    # Failed configuration
                     nsr_obj.vnf_failed = True
-                else:
+                    done = True
                     yield from self.update_vnf_state(vnf_cfg, conmanY.RecordState.CFG_FAILED)
-                    self._log.error("Failed to Apply Pending Configuration for VNF = {}, remaining retries({})"
-                                    .format(log_this_vnf(vnf_cfg), vnf_cfg['cfg_retries']))
+                    self._log.error("Failed to apply configuration for VNF = {}"
+                                    .format(log_this_vnf(vnf_cfg)))
+
             return done
             
         @asyncio.coroutine
@@ -147,6 +163,12 @@ class ConfigurationManager(object):
             nsr_obj.vnf_failed = False
             vnf_cfg_list = nsr_obj.vnf_cfg_list
             while vnf_cfg_list:
+                # Check to make sure the NSR is still valid
+                if nsr_obj.parent.is_nsr_valid(nsr_obj.nsr_id) is False:
+                    self._log.info("NSR {} not found, could be terminated".
+                                    format(nsr_obj.nsr_id))
+                    return
+
                 # Need while loop here, since we will be removing list item
                 vnf_cfg = vnf_cfg_list.pop(0)
                 self._log.info("Applying Pending Configuration for VNF = %s / %s", log_this_vnf(vnf_cfg), vnf_cfg['agent_vnfr'])
@@ -154,21 +176,50 @@ class ConfigurationManager(object):
                 self._log.debug("Applied Pending Configuration for VNF = {}, status={}"
                                 .format(log_this_vnf(vnf_cfg), vnf_done))
                 if not vnf_done:
-                    if vnf_cfg['cfg_retries']:
-                        # We will retry, but we will give other VNF chance first since this one failed.
-                        vnf_cfg_list.append(vnf_cfg)
-                    else:
-                        # Mark that at least one VNF failed
-                        ret_status = False
-                     
+                    # We will retry, but we will give other VNF chance first since this one failed.
+                    vnf_cfg_list.append(vnf_cfg)
+
+            if nsr_obj.vnf_failed:
+                # Mark that at least one VNF failed
+                ret_status = False
+
+            # Set the config status for the NSR
+            if ret_status:
+                yield from nsr_obj.update_ns_cm_state(conmanY.RecordState.READY)
+            elif nsr_obj.vnf_failed or nsr_obj.nsr_failed:
+                yield from nsr_obj.update_ns_cm_state(conmanY.RecordState.CFG_FAILED)
+
             return ret_status
-                    
+
         # Basically, this loop will never end.
         while True:
+            # Check the pending tasks are complete
+            # Store a list of tasks that are completed and
+            # remove from the pending_tasks list outside loop
+            ids = []
+            for nsr_id, task in self.pending_tasks.items():
+                if task.done():
+                    ids.append(nsr_id)
+                    e = task.exception()
+                    if e:
+                        self._log.error("Exception in configuring nsr {}: {}".
+                                        format(nsr_id, e))
+                        self._log.exception(e)
+
+                    else:
+                        rc = task.result()
+                        self._log.debug("NSR {} configured: {}".format(nsr_id, rc))
+                else:
+                    self._log.debug("NSR {} still configuring".format(nsr_id))
+
+            # Remove the completed tasks
+            for nsr_id in ids:
+                self.pending_tasks.pop(nsr_id)
+
             # Sleep before processing any NS (Why are we getting multiple NSR running DTS updates?)
             # If the sleep is not 10 seconds it does not quite work, NSM is marking it 'running' wrongfully 10 seconds in advance?
             yield from asyncio.sleep(10, loop=self._loop)
-                
+
             if self.pending_cfg:
                 # get first NS, pending_cfg is nsr_obj list
                 nsr_obj = self.pending_cfg[0]
@@ -177,17 +228,29 @@ class ConfigurationManager(object):
                     # Process this NS, returns back same obj is successfull or exceeded retries
                     try:
                         self._log.info("Processing NSR:{}".format(nsr_obj.nsr_name))
-                        yield from self.update_ns_state(nsr_obj, conmanY.RecordState.CFG_PROCESS)
-                        nsr_done = yield from process_nsr_obj(nsr_obj)
-                        self._log.info("Process NSR returned: {}".format(nsr_done))
+
+                        # Check if we already have a task running for this NSR
+                        # Case where we are still configuring and terminate is called
+                        if nsr_obj.nsr_id in self.pending_tasks:
+                            self._log.warn("NSR {} in state {} has a configure task running.".
+                                           format(nsr_obj.nsr_name, nsr_obj.get_ns_cm_state()))
+                            # Terminate the task for this NSR
+                            self.pending_tasks[nsr_obj.nsr_id].cancel()
+                            yield from self.update_ns_state(nsr_obj, conmanY.RecordState.CFG_PROCESS)
+
+
+                        # Call in a separate thread
+                        self.pending_tasks[nsr_obj.nsr_id] = \
+                            self._loop.create_task(
+                                    process_nsr_obj(nsr_obj)
+                            )
+
+                        # Remove this nsr_obj
+                        self.pending_cfg.remove(nsr_obj)
+
                     except Exception as e:
                         self._log.error("Failed to process NSR as %s", str(e))
-                    if nsr_done:
-                        yield from self.update_ns_state(nsr_obj, conmanY.RecordState.READY)
-                    elif nsr_obj.vnf_failed:
-                        yield from self.update_ns_state(nsr_obj, conmanY.RecordState.CFG_FAILED)
-                # Remove this nsr_obj only after we are done with it
-                self.pending_cfg.remove(nsr_obj)
+                        self._log.exception(e)
 
     @asyncio.coroutine
     def register(self):
@@ -207,7 +270,6 @@ class ConfigManagerTasklet(rift.tasklets.Tasklet):
 
     def start(self):
         super(ConfigManagerTasklet, self).start()
-        self.log.setLevel(logging.DEBUG)
 
         self.log.debug("Registering with dts")
 
@@ -259,4 +321,4 @@ class ConfigManagerTasklet(rift.tasklets.Tasklet):
         # Transition dts to next state
         next_state = switch.get(state, None)
         if next_state is not None:
-            self._dts.handle.set_state(next_state)
\ No newline at end of file
+            self._dts.handle.set_state(next_state)