rwcm/plugins/rwconman/rift/tasklets/rwconmantasklet/rwconman_config.py

   1
   2 #
   3 #   Copyright 2016 RIFT.IO Inc
   4 #
   5 #   Licensed under the Apache License, Version 2.0 (the "License");
   6 #   you may not use this file except in compliance with the License.
   7 #   You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #   Unless required by applicable law or agreed to in writing, software
  12 #   distributed under the License is distributed on an "AS IS" BASIS,
  13 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #   See the License for the specific language governing permissions and
  15 #   limitations under the License.
  16 #
  17
  18 import asyncio
  19 import os
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import yaml
  25
  26 from gi.repository import (
  27     RwDts as rwdts,
  28     RwConmanYang as conmanY,
  29     ProtobufC,
  30 )
  31
  32 import rift.tasklets
  33 import rift.package.script
  34 import rift.package.store
  35
  36 from . import rwconman_conagent as conagent
  37 from . import RiftCM_rpc
  38 from . import riftcm_config_plugin
  39
  40
  41 if sys.version_info < (3, 4, 4):
  42     asyncio.ensure_future = asyncio.async
  43
  44 def get_vnf_unique_name(nsr_name, vnfr_short_name, member_vnf_index):
  45     return "{}.{}.{}".format(nsr_name, vnfr_short_name, member_vnf_index)
  46
  47 class ConmanConfigError(Exception):
  48     pass
  49
  50
  51 class InitialConfigError(ConmanConfigError):
  52     pass
  53
  54
  55 def log_this_vnf(vnf_cfg):
  56     log_vnf = ""
  57     used_item_list = ['nsr_name', 'vnfr_name', 'member_vnf_index', 'mgmt_ip_address']
  58     for item in used_item_list:
  59         if item in vnf_cfg:
  60             if item == 'mgmt_ip_address':
  61                 log_vnf += "({})".format(vnf_cfg[item])
  62             else:
  63                 log_vnf += "{}/".format(vnf_cfg[item])
  64     return log_vnf
  65
  66 class PretendNsm(object):
  67     def __init__(self, dts, log, loop, parent):
  68         self._dts = dts
  69         self._log = log
  70         self._loop = loop
  71         self._parent = parent
  72         self._nsrs = {}
  73         self._nsr_dict = parent._nsr_dict
  74         self._config_agent_plugins = []
  75         self._nsd_msg = {}
  76
  77     @property
  78     def nsrs(self):
  79         # Expensive, instead use get_nsr, if you know id.
  80         self._nsrs = {}
  81         # Update the list of nsrs (agent nsr)
  82         for id, nsr_obj in self._nsr_dict.items():
  83             self._nsrs[id] = nsr_obj.agent_nsr
  84         return self._nsrs
  85
  86     def get_nsr(self, nsr_id):
  87         if nsr_id in self._nsr_dict:
  88             nsr_obj = self._nsr_dict[nsr_id]
  89             return nsr_obj._nsr
  90         return None
  91
  92     def get_vnfr_msg(self, vnfr_id, nsr_id=None):
  93         self._log.debug("get_vnfr_msg(vnfr=%s, nsr=%s)",
  94                         vnfr_id, nsr_id)
  95         found = False
  96         if nsr_id:
  97             if nsr_id in self._nsr_dict:
  98                 nsr_obj = self._nsr_dict[nsr_id]
  99                 if vnfr_id in nsr_obj._vnfr_dict:
 100                     found = True
 101         else:
 102             for nsr_obj in self._nsr_dict.values():
 103                 if vnfr_id in nsr_obj._vnfr_dict:
 104                     # Found it
 105                     found = True
 106                     break
 107         if found:
 108             vnf_cfg = nsr_obj._vnfr_dict[vnfr_id]['vnf_cfg']
 109             return vnf_cfg['agent_vnfr'].vnfr_msg
 110         else:
 111             return None
 112
 113     @asyncio.coroutine
 114     def get_nsd(self, nsr_id):
 115         if nsr_id not in self._nsd_msg:
 116             nsr_config = yield from self._parent.cmdts_obj.get_nsr_config(nsr_id)
 117             self._nsd_msg[nsr_id] = nsr_config.nsd
 118         return self._nsd_msg[nsr_id]
 119
 120     @property
 121     def config_agent_plugins(self):
 122         self._config_agent_plugins = []
 123         for agent in self._parent._config_agent_mgr._plugin_instances.values():
 124             self._config_agent_plugins.append(agent)
 125         return self._config_agent_plugins
 126
 127 class ConfigManagerConfig(object):
 128     def __init__(self, dts, log, loop, parent):
 129         self._dts = dts
 130         self._log = log
 131         self._loop = loop
 132         self._parent = parent
 133         self._nsr_dict = {}
 134         self.pending_cfg = {}
 135         self.terminate_cfg = {}
 136         self.pending_tasks = [] # User for NSRid get retry
 137                                 # (mainly excercised at restart case)
 138         self._config_xpath = "C,/cm-config"
 139         self._opdata_xpath = "D,/rw-conman:cm-state"
 140
 141         self.cm_config = conmanY.SoConfig()
 142         # RO specific configuration
 143         self.ro_config = {}
 144         for key in self.cm_config.ro_endpoint.fields:
 145             self.ro_config[key] = None
 146
 147         # Initialize cm-state
 148         self.cm_state = {}
 149         self.cm_state['cm_nsr'] = []
 150         self.cm_state['states'] = "Initialized"
 151
 152         # Initialize objects to register
 153         self.cmdts_obj = ConfigManagerDTS(self._log, self._loop, self, self._dts)
 154         self._config_agent_mgr = conagent.RiftCMConfigAgent(
 155             self._dts,
 156             self._log,
 157             self._loop,
 158             self,
 159         )
 160         self.reg_handles = [
 161             self.cmdts_obj,
 162             self._config_agent_mgr,
 163             RiftCM_rpc.RiftCMRPCHandler(self._dts, self._log, self._loop,
 164                                         PretendNsm(
 165                                             self._dts, self._log, self._loop, self)),
 166         ]
 167
 168     def is_nsr_valid(self, nsr_id):
 169         if nsr_id in self._nsr_dict:
 170             return True
 171         return False
 172
 173     def add_to_pending_tasks(self, task):
 174         if self.pending_tasks:
 175             for p_task in self.pending_tasks:
 176                 if p_task['nsrid'] == task['nsrid']:
 177                     # Already queued
 178                     return
 179         try:
 180             self.pending_tasks.append(task)
 181             self._log.debug("add_to_pending_tasks (nsrid:%s)",
 182                             task['nsrid'])
 183             if len(self.pending_tasks) == 1:
 184                 self._loop.create_task(self.ConfigManagerConfig_pending_loop())
 185                 # TBD - change to info level
 186                 self._log.debug("Started pending_loop!")
 187         except Exception as e:
 188             self._log.error("Failed adding to pending tasks (%s)", str(e))
 189
 190     def del_from_pending_tasks(self, task):
 191         try:
 192             self.pending_tasks.remove(task)
 193         except Exception as e:
 194             self._log.error("Failed removing from pending tasks (%s)", str(e))
 195
 196     @asyncio.coroutine
 197     def ConfigManagerConfig_pending_loop(self):
 198         loop_sleep = 2
 199         while True:
 200             yield from asyncio.sleep(loop_sleep, loop=self._loop)
 201             """
 202             This pending task queue is ordred by events,
 203             must finish previous task successfully to be able to go on to the next task
 204             """
 205             if self.pending_tasks:
 206                 self._log.debug("self.pending_tasks len=%s", len(self.pending_tasks))
 207                 task = self.pending_tasks[0]
 208                 done = False
 209                 if 'nsrid' in task:
 210                     nsrid = task['nsrid']
 211                     self._log.debug("Will execute pending task for NSR id(%s)", nsrid)
 212                     try:
 213                         # Try to configure this NSR
 214                         task['retries'] -= 1
 215                         done = yield from self.config_NSR(nsrid)
 216                         self._log.info("self.config_NSR status=%s", done)
 217
 218                     except Exception as e:
 219                         self._log.error("Failed(%s) configuring NSR(%s)," \
 220                                         "retries remained:%d!",
 221                                         str(e), nsrid, task['retries'])
 222                     finally:
 223                         self.pending_tasks.remove(task)
 224
 225                     if done:
 226                         self._log.debug("Finished pending task NSR id(%s):", nsrid)
 227                     else:
 228                         self._log.error("Failed configuring NSR(%s), retries remained:%d!",
 229                                         nsrid, task['retries'])
 230
 231                         # Failed, re-insert (append at the end)
 232                         # this failed task to be retried later
 233                         # If any retries remained.
 234                         if task['retries']:
 235                             self.pending_tasks.append(task)
 236             else:
 237                 self._log.debug("Stopped pending_loop!")
 238                 break
 239
 240     @asyncio.coroutine
 241     def register(self):
 242         yield from self.register_cm_state_opdata()
 243
 244         # Initialize all handles that needs to be registered
 245         for reg in self.reg_handles:
 246             yield from reg.register()
 247
 248     @asyncio.coroutine
 249     def register_cm_state_opdata(self):
 250
 251         def state_to_string(state):
 252             state_dict = {
 253                 conmanY.RecordState.INIT : "init",
 254                 conmanY.RecordState.RECEIVED : "received",
 255                 conmanY.RecordState.CFG_PROCESS : "cfg_process",
 256                 conmanY.RecordState.CFG_PROCESS_FAILED : "cfg_process_failed",
 257                 conmanY.RecordState.CFG_SCHED : "cfg_sched",
 258                 conmanY.RecordState.CFG_DELAY : "cfg_delay",
 259                 conmanY.RecordState.CONNECTING : "connecting",
 260                 conmanY.RecordState.FAILED_CONNECTION : "failed_connection",
 261                 conmanY.RecordState.NETCONF_CONNECTED : "netconf_connected",
 262                 conmanY.RecordState.NETCONF_SSH_CONNECTED : "netconf_ssh_connected",
 263                 conmanY.RecordState.RESTCONF_CONNECTED : "restconf_connected",
 264                 conmanY.RecordState.CFG_SEND : "cfg_send",
 265                 conmanY.RecordState.CFG_FAILED : "cfg_failed",
 266                 conmanY.RecordState.READY_NO_CFG : "ready_no_cfg",
 267                 conmanY.RecordState.READY : "ready",
 268                 }
 269             return state_dict[state]
 270
 271         @asyncio.coroutine
 272         def on_prepare(xact_info, action, ks_path, msg):
 273
 274             self._log.debug("Received cm-state: msg=%s, action=%s", msg, action)
 275
 276             if action == rwdts.QueryAction.READ:
 277                 show_output = conmanY.CmOpdata()
 278                 show_output.from_dict(self.cm_state)
 279                 self._log.debug("Responding to SHOW cm-state: %s", self.cm_state)
 280                 xact_info.respond_xpath(rwdts.XactRspCode.ACK,
 281                                         xpath=self._opdata_xpath,
 282                                         msg=show_output)
 283             else:
 284                 xact_info.respond_xpath(rwdts.XactRspCode.ACK)
 285
 286         self._log.info("Registering for cm-opdata xpath: %s",
 287                         self._opdata_xpath)
 288
 289         try:
 290             handler=rift.tasklets.DTS.RegistrationHandler(on_prepare=on_prepare)
 291             yield from self._dts.register(xpath=self._opdata_xpath,
 292                                           handler=handler,
 293                                           flags=rwdts.Flag.PUBLISHER)
 294             self._log.info("Successfully registered for opdata(%s)", self._opdata_xpath)
 295         except Exception as e:
 296             self._log.error("Failed to register for opdata as (%s)", e)
 297
 298     @asyncio.coroutine
 299     def process_nsd_vnf_configuration(self, nsr_obj, vnfr):
 300
 301         def get_config_method(vnf_config):
 302             cfg_types = ['netconf', 'juju', 'script']
 303             for method in cfg_types:
 304                 if method in vnf_config:
 305                     return method
 306             return None
 307
 308         def get_cfg_file_extension(method,  configuration_options):
 309             ext_dict = {
 310                 "netconf" : "xml",
 311                 "script" : {
 312                     "bash" : "sh",
 313                     "expect" : "exp",
 314                 },
 315                 "juju" : "yml"
 316             }
 317
 318             if method == "netconf":
 319                 return ext_dict[method]
 320             elif method == "script":
 321                 return ext_dict[method][configuration_options['script_type']]
 322             elif method == "juju":
 323                 return ext_dict[method]
 324             else:
 325                 return "cfg"
 326
 327         # This is how the YAML file should look like,
 328         # This routine will be called for each VNF, so keep appending the file.
 329         # priority order is determined by the number,
 330         # hence no need to generate the file in that order. A dictionary will be
 331         # used that will take care of the order by number.
 332         '''
 333         1 : <== This is priority
 334           name : trafsink_vnfd
 335           member_vnf_index : 2
 336           configuration_delay : 120
 337           configuration_type : netconf
 338           configuration_options :
 339             username : admin
 340             password : admin
 341             port : 2022
 342             target : running
 343         2 :
 344           name : trafgen_vnfd
 345           member_vnf_index : 1
 346           configuration_delay : 0
 347           configuration_type : netconf
 348           configuration_options :
 349             username : admin
 350             password : admin
 351             port : 2022
 352             target : running
 353         '''
 354
 355         # Save some parameters needed as short cuts in flat structure (Also generated)
 356         vnf_cfg = vnfr['vnf_cfg']
 357         # Prepare unique name for this VNF
 358         vnf_cfg['vnf_unique_name'] = get_vnf_unique_name(
 359             vnf_cfg['nsr_name'], vnfr['short_name'], vnfr['member_vnf_index_ref'])
 360
 361         nsr_obj.cfg_path_prefix = '{}/{}_{}'.format(
 362             nsr_obj.this_nsr_dir, vnfr['short_name'], vnfr['member_vnf_index_ref'])
 363         nsr_vnfr = '{}/{}_{}'.format(
 364             vnf_cfg['nsr_name'], vnfr['short_name'], vnfr['member_vnf_index_ref'])
 365
 366         # Get vnf_configuration from vnfr
 367         vnf_config = vnfr['vnf_configuration']
 368
 369         self._log.debug("vnf_configuration = %s", vnf_config)
 370
 371         # Create priority dictionary
 372         cfg_priority_order = 0
 373         if ('config_attributes' in vnf_config and
 374             'config_priority' in vnf_config['config_attributes']):
 375             cfg_priority_order = vnf_config['config_attributes']['config_priority']
 376
 377         if cfg_priority_order not in nsr_obj.nsr_cfg_config_attributes_dict:
 378             # No VNFR with this priority yet, initialize the list
 379             nsr_obj.nsr_cfg_config_attributes_dict[cfg_priority_order] = []
 380
 381         method = get_config_method(vnf_config)
 382         if method is not None:
 383             # Create all sub dictionaries first
 384             config_priority = {
 385                 'id' : vnfr['id'],
 386                 'name' : vnfr['short_name'],
 387                 'member_vnf_index' : vnfr['member_vnf_index_ref'],
 388             }
 389
 390             if 'config_delay' in vnf_config['config_attributes']:
 391                 config_priority['configuration_delay'] = vnf_config['config_attributes']['config_delay']
 392                 vnf_cfg['config_delay'] = config_priority['configuration_delay']
 393
 394             configuration_options = {}
 395             self._log.debug("config method=%s", method)
 396             config_priority['configuration_type'] = method
 397             vnf_cfg['config_method'] = method
 398
 399             # Set config agent based on method
 400             self._config_agent_mgr.set_config_agent(
 401                 nsr_obj.agent_nsr, vnf_cfg['agent_vnfr'], method)
 402
 403             cfg_opt_list = [
 404                 'port', 'target', 'script_type', 'ip_address', 'user', 'secret',
 405             ]
 406             for cfg_opt in cfg_opt_list:
 407                 if cfg_opt in vnf_config[method]:
 408                     configuration_options[cfg_opt] = vnf_config[method][cfg_opt]
 409                     vnf_cfg[cfg_opt] = configuration_options[cfg_opt]
 410
 411             cfg_opt_list = ['mgmt_ip_address', 'username', 'password']
 412             for cfg_opt in cfg_opt_list:
 413                 if cfg_opt in vnf_config['config_access']:
 414                     configuration_options[cfg_opt] = vnf_config['config_access'][cfg_opt]
 415                     vnf_cfg[cfg_opt] = configuration_options[cfg_opt]
 416
 417             # Add to the cp_dict
 418             vnf_cp_dict = nsr_obj._cp_dict[vnfr['member_vnf_index_ref']]
 419             vnf_cp_dict['rw_mgmt_ip'] = vnf_cfg['mgmt_ip_address']
 420             vnf_cp_dict['rw_username'] = vnf_cfg['username']
 421             vnf_cp_dict['rw_password'] = vnf_cfg['password']
 422
 423
 424             # TBD - see if we can neatly include the config in "config_attributes" file, no need though
 425             #config_priority['config_template'] = vnf_config['config_template']
 426             # Create config file
 427             vnf_cfg['juju_script'] = os.path.join(self._parent.cfg_dir, 'juju_if.py')
 428
 429             if 'config_template' in vnf_config:
 430                 vnf_cfg['cfg_template'] = '{}_{}_template.cfg'.format(nsr_obj.cfg_path_prefix, config_priority['configuration_type'])
 431                 vnf_cfg['cfg_file'] = '{}.{}'.format(nsr_obj.cfg_path_prefix, get_cfg_file_extension(method, configuration_options))
 432                 vnf_cfg['xlate_script'] = os.path.join(self._parent.cfg_dir, 'xlate_cfg.py')
 433                 try:
 434                     # Now write this template into file
 435                     with open(vnf_cfg['cfg_template'], "w") as cf:
 436                         cf.write(vnf_config['config_template'])
 437                 except Exception as e:
 438                     self._log.error("Processing NSD, failed to generate configuration template : %s (Error : %s)",
 439                                     vnf_config['config_template'], str(e))
 440                     raise
 441
 442             self._log.debug("VNF endpoint so far: %s", vnf_cfg)
 443
 444             # Populate filled up dictionary
 445             config_priority['configuration_options'] = configuration_options
 446             nsr_obj.nsr_cfg_config_attributes_dict[cfg_priority_order].append(config_priority)
 447             nsr_obj.num_vnfs_to_cfg += 1
 448             nsr_obj._vnfr_dict[vnf_cfg['vnf_unique_name']] = vnfr
 449             nsr_obj._vnfr_dict[vnfr['id']] = vnfr
 450
 451             self._log.debug("VNF:(%s) config_attributes = %s",
 452                             log_this_vnf(vnfr['vnf_cfg']),
 453                             nsr_obj.nsr_cfg_config_attributes_dict)
 454         else:
 455             self._log.info("VNF:(%s) is not to be configured by Configuration Manager!",
 456                            log_this_vnf(vnfr['vnf_cfg']))
 457             yield from nsr_obj.update_vnf_cm_state(vnfr, conmanY.RecordState.READY_NO_CFG)
 458
 459         # Update the cm-state
 460         nsr_obj.populate_vm_state_from_vnf_cfg()
 461
 462     @asyncio.coroutine
 463     def config_NSR(self, id):
 464
 465         def my_yaml_dump(config_attributes_dict, yf):
 466
 467             yaml_dict = dict(sorted(config_attributes_dict.items()))
 468             yf.write(yaml.dump(yaml_dict, default_flow_style=False))
 469
 470         nsr_dict = self._nsr_dict
 471         self._log.info("Configure NSR, id = %s", id)
 472
 473         #####################TBD###########################
 474         # yield from self._config_agent_mgr.invoke_config_agent_plugins('notify_create_nsr', self.id, self._nsd)
 475         # yield from self._config_agent_mgr.invoke_config_agent_plugins('notify_nsr_active', self.id, self._vnfrs)
 476
 477         try:
 478             if id not in nsr_dict:
 479                 nsr_obj = ConfigManagerNSR(self._log, self._loop, self, id)
 480                 nsr_dict[id] = nsr_obj
 481             else:
 482                 self._log.info("NSR(%s) is already initialized!", id)
 483                 nsr_obj = nsr_dict[id]
 484         except Exception as e:
 485             self._log.error("Failed creating NSR object for (%s) as (%s)", id, str(e))
 486             raise
 487
 488         # Try to configure this NSR only if not already processed
 489         if nsr_obj.cm_nsr['state'] != nsr_obj.state_to_string(conmanY.RecordState.INIT):
 490             self._log.debug("NSR(%s) is already processed, state=%s",
 491                             nsr_obj.nsr_name, nsr_obj.cm_nsr['state'])
 492             yield from nsr_obj.publish_cm_state()
 493             return True
 494
 495         cmdts_obj = self.cmdts_obj
 496         try:
 497             # Fetch NSR
 498             nsr = yield from cmdts_obj.get_nsr(id)
 499             self._log.debug("Full NSR : %s", nsr)
 500             if nsr['operational_status'] != "running":
 501                 self._log.info("NSR(%s) is not ready yet!", nsr['nsd_name_ref'])
 502                 return False
 503             self._nsr = nsr
 504
 505             # Create Agent NSR class
 506             nsr_config = yield from cmdts_obj.get_nsr_config(id)
 507             self._log.debug("NSR {} config: {}".format(id, nsr_config))
 508             nsr_obj.agent_nsr = riftcm_config_plugin.RiftCMnsr(nsr, nsr_config)
 509
 510             try:
 511                 yield from nsr_obj.update_ns_cm_state(conmanY.RecordState.RECEIVED)
 512
 513                 # Parse NSR
 514                 if nsr is not None:
 515                     nsr_obj.set_nsr_name(nsr['name_ref'])
 516                     nsr_dir = os.path.join(self._parent.cfg_dir, nsr_obj.nsr_name)
 517                     self._log.info("Checking NS config directory: %s", nsr_dir)
 518                     if not os.path.isdir(nsr_dir):
 519                         os.makedirs(nsr_dir)
 520                         # self._log.critical("NS %s is not to be configured by Service Orchestrator!", nsr_obj.nsr_name)
 521                         # yield from nsr_obj.update_ns_cm_state(conmanY.RecordState.READY_NO_CFG)
 522                         # return
 523
 524                     nsr_obj.set_config_dir(self)
 525
 526                     for const_vnfr in nsr['constituent_vnfr_ref']:
 527                         self._log.debug("Fetching VNFR (%s)", const_vnfr['vnfr_id'])
 528                         vnfr_msg = yield from cmdts_obj.get_vnfr(const_vnfr['vnfr_id'])
 529                         if vnfr_msg:
 530                             vnfr = vnfr_msg.as_dict()
 531                             self._log.info("create VNF:{}/{}".format(nsr_obj.nsr_name, vnfr['short_name']))
 532                             agent_vnfr = yield from nsr_obj.add_vnfr(vnfr, vnfr_msg)
 533
 534                             # Preserve order, self.process_nsd_vnf_configuration()
 535                             # sets up the config agent based on the method
 536                             yield from self.process_nsd_vnf_configuration(nsr_obj, vnfr)
 537                             yield from self._config_agent_mgr.invoke_config_agent_plugins(
 538                                 'notify_create_vnfr',
 539                                 nsr_obj.agent_nsr,
 540                                 agent_vnfr)
 541
 542                         #####################TBD###########################
 543                         # self._log.debug("VNF active. Apply initial config for vnfr {}".format(vnfr.name))
 544                         # yield from self._config_agent_mgr.invoke_config_agent_plugins('apply_initial_config',
 545                         #                                             vnfr.id, vnfr)
 546                         # yield from self._config_agent_mgr.invoke_config_agent_plugins('notify_terminate_vnf', self.id, vnfr)
 547
 548             except Exception as e:
 549                 self._log.error("Failed processing NSR (%s) as (%s)", nsr_obj.nsr_name, str(e))
 550                 self._log.exception(e)
 551                 yield from nsr_obj.update_ns_cm_state(conmanY.RecordState.CFG_PROCESS_FAILED)
 552                 raise
 553
 554             try:
 555                 # Generate config_config_attributes.yaml (For debug reference)
 556                 with open(nsr_obj.config_attributes_file, "w") as yf:
 557                     my_yaml_dump(nsr_obj.nsr_cfg_config_attributes_dict, yf)
 558             except Exception as e:
 559                 self._log.error("NS:(%s) failed to write config attributes file as (%s)", nsr_obj.nsr_name, str(e))
 560
 561             try:
 562                 # Generate nsr_xlate_dict.yaml (For debug reference)
 563                 with open(nsr_obj.xlate_dict_file, "w") as yf:
 564                     yf.write(yaml.dump(nsr_obj._cp_dict, default_flow_style=False))
 565             except Exception as e:
 566                 self._log.error("NS:(%s) failed to write nsr xlate tags file as (%s)", nsr_obj.nsr_name, str(e))
 567
 568             self._log.debug("Starting to configure each VNF")
 569
 570             # Check if this NS has input parametrs
 571             self._log.info("Checking NS configuration order: %s", nsr_obj.config_attributes_file)
 572
 573             if os.path.exists(nsr_obj.config_attributes_file):
 574                 # Apply configuration is specified order
 575                 try:
 576                     # Go in loop to configure by specified order
 577                     self._log.info("Using Dynamic configuration input parametrs for NS: %s", nsr_obj.nsr_name)
 578
 579                     # cfg_delay = nsr_obj.nsr_cfg_config_attributes_dict['configuration_delay']
 580                     # if cfg_delay:
 581                     #     self._log.info("Applying configuration delay for NS (%s) ; %d seconds",
 582                     #                    nsr_obj.nsr_name, cfg_delay)
 583                     #     yield from asyncio.sleep(cfg_delay, loop=self._loop)
 584
 585                     for config_attributes_dict in nsr_obj.nsr_cfg_config_attributes_dict.values():
 586                         # Iterate through each priority level
 587                         for vnf_config_attributes_dict in config_attributes_dict:
 588                             # Iterate through each vnfr at this priority level
 589
 590                             # Make up vnf_unique_name with vnfd name and member index
 591                             #vnfr_name = "{}.{}".format(nsr_obj.nsr_name, vnf_config_attributes_dict['name'])
 592                             vnf_unique_name = get_vnf_unique_name(
 593                                 nsr_obj.nsr_name,
 594                                 vnf_config_attributes_dict['name'],
 595                                 str(vnf_config_attributes_dict['member_vnf_index']),
 596                             )
 597                             self._log.info("NS (%s) : VNF (%s) - Processing configuration attributes",
 598                                            nsr_obj.nsr_name, vnf_unique_name)
 599
 600                             # Find vnfr for this vnf_unique_name
 601                             if vnf_unique_name not in nsr_obj._vnfr_dict:
 602                                 self._log.error("NS (%s) - Can not find VNF to be configured: %s", nsr_obj.nsr_name, vnf_unique_name)
 603                             else:
 604                                 # Save this unique VNF's config input parameters
 605                                 nsr_obj.vnf_config_attributes_dict[vnf_unique_name] = vnf_config_attributes_dict
 606                                 nsr_obj.ConfigVNF(nsr_obj._vnfr_dict[vnf_unique_name])
 607
 608                     # Now add the entire NS to the pending config list.
 609                     self._log.info("Scheduling NSR:{} configuration".format(nsr_obj.nsr_name))
 610                     self._parent.add_to_pending(nsr_obj)
 611                     self._parent.add_nsr_obj(nsr_obj)
 612
 613                 except Exception as e:
 614                     self._log.error("Failed processing input parameters for NS (%s) as %s", nsr_obj.nsr_name, str(e))
 615                     raise
 616             else:
 617                 self._log.error("No configuration input parameters for NSR (%s)", nsr_obj.nsr_name)
 618
 619         except Exception as e:
 620             self._log.error("Failed to configure NS (%s) as (%s)", nsr_obj.nsr_name, str(e))
 621             yield from nsr_obj.update_ns_cm_state(conmanY.RecordState.CFG_PROCESS_FAILED)
 622             raise
 623
 624         return True
 625
 626     @asyncio.coroutine
 627     def terminate_NSR(self, id):
 628         nsr_dict = self._nsr_dict
 629         if id not in nsr_dict:
 630             self._log.error("NSR(%s) does not exist!", id)
 631             return
 632         else:
 633             # Remove this NSR if we have it on pending task list
 634             for task in self.pending_tasks:
 635                 if task['nsrid'] == id:
 636                     self.del_from_pending_tasks(task)
 637
 638             # Remove this object from global list
 639             nsr_obj = nsr_dict.pop(id, None)
 640
 641             # Remove this NS cm-state from global status list
 642             self.cm_state['cm_nsr'].remove(nsr_obj.cm_nsr)
 643
 644             # Also remove any scheduled configuration event
 645             for nsr_obj_p in self._parent.pending_cfg:
 646                 if nsr_obj_p == nsr_obj:
 647                     assert id == nsr_obj_p._nsr_id
 648                     #self._parent.pending_cfg.remove(nsr_obj_p)
 649                     # Mark this as being deleted so we do not try to configure it if we are in cfg_delay (will wake up and continue to process otherwise)
 650                     nsr_obj_p.being_deleted = True
 651                     self._log.info("Removed scheduled configuration for NSR(%s)", nsr_obj.nsr_name)
 652
 653             self._parent.remove_nsr_obj(id)
 654
 655             # Call Config Agent to clean up for each VNF
 656             for agent_vnfr in nsr_obj.agent_nsr.vnfrs:
 657                 yield from self._config_agent_mgr.invoke_config_agent_plugins(
 658                     'notify_terminate_vnfr',
 659                     nsr_obj.agent_nsr,
 660                     agent_vnfr)
 661
 662             # publish delete cm-state (cm-nsr)
 663             yield from nsr_obj.delete_cm_nsr()
 664
 665             #####################TBD###########################
 666             # yield from self._config_agent_mgr.invoke_config_agent_plugins('notify_terminate_ns', self.id)
 667
 668             self._log.info("NSR(%s/%s) is deleted", nsr_obj.nsr_name, id)
 669
 670     @asyncio.coroutine
 671     def process_initial_config(self, nsr_obj, conf, script, vnfr_name=None):
 672         '''Apply the initial-config-primitives specified in NSD or VNFD'''
 673
 674         def get_input_file(parameters):
 675             inp = {}
 676
 677             # Add NSR name to file
 678             inp['nsr_name'] = nsr_obj.nsr_name
 679
 680             # Add VNFR name if available
 681             if vnfr_name:
 682                 inp['vnfr_name'] = vnfr_name
 683
 684             # Add parameters for initial config
 685             inp['parameter'] = {}
 686             for parameter in parameters:
 687                 try:
 688                     inp['parameter'][parameter['name']] = parameter['value']
 689                 except KeyError as e:
 690                     if vnfr_name:
 691                         self._log.info("VNFR {} initial config parameter {} with no value: {}".
 692                                        format(vnfr_name, parameter, e))
 693                     else:
 694                         self._log.info("NSR {} initial config parameter {} with no value: {}".
 695                                        format(nsr_obj.nsr_name, parameter, e))
 696
 697
 698             # Add config agents specific to each VNFR
 699             inp['config-agent'] = {}
 700             for vnfr in nsr_obj.agent_nsr.vnfrs:
 701                 # Get the config agent for the VNFR
 702                 # If vnfr name is specified, add only CA specific to that
 703                 if (vnfr_name is None) or \
 704                    (vnfr_name == vnfr.name):
 705                     agent = self._config_agent_mgr.get_vnfr_config_agent(vnfr.vnfr_msg)
 706                     if agent:
 707                         if agent.agent_type != riftcm_config_plugin.DEFAULT_CAP_TYPE:
 708                             inp['config-agent'][vnfr.member_vnf_index] = agent.agent_data
 709                             inp['config-agent'][vnfr.member_vnf_index] \
 710                                 ['service-name'] = agent.get_service_name(vnfr.id)
 711
 712             # Add vnfrs specific data
 713             inp['vnfr'] = {}
 714             for vnfr in nsr_obj.vnfrs:
 715                 v = {}
 716
 717                 v['name'] = vnfr['name']
 718                 v['mgmt_ip_address'] = vnfr['vnf_cfg']['mgmt_ip_address']
 719                 v['mgmt_port'] = vnfr['vnf_cfg']['port']
 720
 721                 if 'dashboard_url' in vnfr:
 722                     v['dashboard_url'] = vnfr['dashboard_url']
 723
 724                 if 'connection_point' in vnfr:
 725                     v['connection_point'] = []
 726                     for cp in vnfr['connection_point']:
 727                         v['connection_point'].append(
 728                             {
 729                                 'name': cp['name'],
 730                                 'ip_address': cp['ip_address'],
 731                             }
 732                         )
 733
 734                 v['vdur'] = []
 735                 vdu_data = []
 736                 for vdu in vnfr['vdur']:
 737                     d = {}
 738                     for k in ['name','management_ip', 'vm_management_ip', 'id', 'vdu_id_ref']:
 739                         if k in vdu:
 740                             d[k] = vdu[k]
 741                     vdu_data.append(d)
 742                 v['vdur'] = vdu_data
 743
 744                 inp['vnfr'][vnfr['member_vnf_index_ref']] = v
 745
 746
 747             self._log.debug("Input data for {}: {}".
 748                             format((vnfr_name if vnfr_name else nsr_obj.nsr_name),
 749                                    inp))
 750
 751             # Convert to YAML string
 752             yaml_string = yaml.dump(inp, default_flow_style=False)
 753
 754             # Write the inputs as yaml file
 755             tmp_file = None
 756             with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
 757                 tmp_file.write(yaml_string.encode("UTF-8"))
 758             self._log.debug("Input file created for {}: {}".
 759                             format((vnfr_name if vnfr_name \
 760                                     else nsr_obj.nsr_name),
 761                                    tmp_file.name))
 762
 763             return tmp_file.name
 764
 765         parameters = []
 766         try:
 767             parameters = conf['parameter']
 768         except Exception as e:
 769             self._log.debug("Parameter conf: {}, e: {}".
 770                             format(conf, e))
 771
 772         inp_file = get_input_file(parameters)
 773
 774         cmd = "{0} {1}".format(script, inp_file)
 775         self._log.debug("Running the CMD: {}".format(cmd))
 776
 777         process = yield from asyncio.create_subprocess_shell(cmd,
 778                                                              loop=self._loop,
 779                                                              stdout=subprocess.PIPE,
 780                                                              stderr=subprocess.PIPE)
 781         stdout, stderr = yield from process.communicate()
 782         rc = yield from process.wait()
 783
 784         if rc:
 785             msg = "NSR/VNFR {} initial config using {} failed with {}: {}". \
 786                   format(vnfr_name if vnfr_name else nsr_obj.nsr_name,
 787                          script, rc, stderr)
 788             self._log.error(msg)
 789             raise InitialConfigError(msg)
 790
 791         try:
 792             os.remove(inp_file)
 793         except Exception as e:
 794             self._log.debug("Error removing input file {}: {}".
 795                             format(inp_file, e))
 796
 797     def get_script_file(self, script_name, d_name, d_id, d_type):
 798         if d_type == "vnfd":
 799             package_store = rift.package.store.VnfdPackageFilesystemStore(self._log)
 800             package_store.refresh()
 801         elif d_type == "nsd":
 802             package_store = rift.package.store.NsdPackageFilesystemStore(self._log)
 803             package_store.refresh()
 804         else:
 805             raise
 806         script_extractor = rift.package.script.PackageScriptExtractor(self._log)
 807         script = script_extractor.get_extracted_script_path(d_id, script_name)
 808
 809         self._log.debug("Checking for script at %s", script)
 810         if not os.path.exists(script):
 811             self._log.warning("Did not find script %s", script)
 812             return
 813
 814         # Seen cases in jenkins, where the script execution fails
 815         # with permission denied. Setting the permission on script
 816         # to make sure it has execute permission
 817         perm = os.stat(script).st_mode
 818         if not (perm  &  stat.S_IXUSR):
 819             self._log.warning("NSR/VNFR {} script {} " \
 820                                     "without execute permission: {}".
 821                                     format(d_name, script, perm))
 822             os.chmod(script, perm | stat.S_IXUSR)
 823         return script
 824
 825     @asyncio.coroutine
 826     def process_ns_initial_config(self, nsr_obj):
 827         '''Apply the initial-config-primitives specified in NSD'''
 828
 829         nsr = yield from self.cmdts_obj.get_nsr(nsr_obj.nsr_id)
 830         if 'initial_config_primitive' not in nsr:
 831             return
 832
 833         if nsr is not None:
 834             nsd = yield from self.cmdts_obj.get_nsd(nsr_obj.nsr_id)
 835             for conf in nsr['initial_config_primitive']:
 836                 self._log.debug("NSR {} initial config: {}".
 837                                 format(nsr_obj.nsr_name, conf))
 838                 script = self.get_script_file(conf['user_defined_script'],
 839                                               nsd.name,
 840                                               nsd.id,
 841                                               'nsd')
 842
 843                 yield from self.process_initial_config(nsr_obj, conf, script)
 844
 845     @asyncio.coroutine
 846     def process_vnf_initial_config(self, nsr_obj, vnfr):
 847         '''Apply the initial-config-primitives specified in VNFD'''
 848
 849         vnfr_name = vnfr.name
 850
 851         vnfd = vnfr.vnfd
 852         vnf_cfg = vnfd.vnf_configuration
 853
 854         for conf in vnf_cfg.initial_config_primitive:
 855                 self._log.debug("VNFR {} initial config: {} for vnfd id {}".
 856                                 format(vnfr_name, conf, vnfd.id))
 857
 858                 if not conf.user_defined_script:
 859                     self._log.debug("VNFR {} did not find user defined script: {}".
 860                                     format(vnfr_name, conf))
 861                     continue
 862
 863                 script = self.get_script_file(conf.user_defined_script,
 864                                               vnfd.name,
 865                                               vnfd.id,
 866                                               'vnfd')
 867
 868                 yield from self.process_initial_config(nsr_obj,
 869                                                        conf.as_dict(),
 870                                                        script,
 871                                                        vnfr_name=vnfr_name)
 872
 873
 874 class ConfigManagerNSR(object):
 875     def __init__(self, log, loop, parent, id):
 876         self._log = log
 877         self._loop = loop
 878         self._rwcal = None
 879         self._vnfr_dict = {}
 880         self._cp_dict = {}
 881         self._nsr_id = id
 882         self._parent = parent
 883         self._log.info("Instantiated NSR entry for id=%s", id)
 884         self.nsr_cfg_config_attributes_dict = {}
 885         self.vnf_config_attributes_dict = {}
 886         self.num_vnfs_to_cfg = 0
 887         self._vnfr_list = []
 888         self.vnf_cfg_list = []
 889         self.this_nsr_dir = None
 890         self.being_deleted = False
 891         self.dts_obj = self._parent.cmdts_obj
 892
 893         # Initialize cm-state for this NS
 894         self.cm_nsr = {}
 895         self.cm_nsr['cm_vnfr'] = []
 896         self.cm_nsr['id'] = id
 897         self.cm_nsr['state'] = self.state_to_string(conmanY.RecordState.INIT)
 898         self.cm_nsr['state_details'] = None
 899
 900         self.set_nsr_name('Not Set')
 901
 902         # Add this NSR cm-state object to global cm-state
 903         parent.cm_state['cm_nsr'].append(self.cm_nsr)
 904
 905         # Place holders for NSR & VNFR classes
 906         self.agent_nsr = None
 907
 908     @property
 909     def nsr_opdata_xpath(self):
 910         ''' Returns full xpath for this NSR cm-state opdata '''
 911         return(
 912             "D,/rw-conman:cm-state" +
 913             "/rw-conman:cm-nsr[rw-conman:id='{}']"
 914         ).format(self._nsr_id)
 915
 916     @property
 917     def vnfrs(self):
 918         return self._vnfr_list
 919
 920     @property
 921     def parent(self):
 922         return self._parent
 923
 924     @property
 925     def nsr_id(self):
 926         return self._nsr_id
 927
 928     @asyncio.coroutine
 929     def publish_cm_state(self):
 930         ''' This function publishes cm_state for this NSR '''
 931
 932         cm_state = conmanY.CmOpdata()
 933         cm_state_nsr = cm_state.cm_nsr.add()
 934         cm_state_nsr.from_dict(self.cm_nsr)
 935         #with self._dts.transaction() as xact:
 936         yield from self.dts_obj.update(self.nsr_opdata_xpath, cm_state_nsr)
 937         self._log.info("Published cm-state with xpath %s and nsr %s",
 938                        self.nsr_opdata_xpath,
 939                        cm_state_nsr)
 940
 941     @asyncio.coroutine
 942     def delete_cm_nsr(self):
 943         ''' This function publishes cm_state for this NSR '''
 944
 945         yield from self.dts_obj.delete(self.nsr_opdata_xpath)
 946         self._log.info("Deleted cm-nsr with xpath %s",
 947                        self.nsr_opdata_xpath)
 948
 949     def set_nsr_name(self, name):
 950         self.nsr_name = name
 951         self.cm_nsr['name'] = name
 952
 953     def set_config_dir(self, caller):
 954         self.this_nsr_dir = os.path.join(
 955             caller._parent.cfg_dir, self.nsr_name, caller._nsr['name_ref'])
 956         if not os.path.exists(self.this_nsr_dir):
 957             os.makedirs(self.this_nsr_dir)
 958             self._log.debug("NSR:(%s), Created configuration directory(%s)",
 959                             caller._nsr['name_ref'], self.this_nsr_dir)
 960         self.config_attributes_file = os.path.join(self.this_nsr_dir, "configuration_config_attributes.yml")
 961         self.xlate_dict_file = os.path.join(self.this_nsr_dir, "nsr_xlate_dict.yml")
 962
 963     def xlate_conf(self, vnfr, vnf_cfg):
 964
 965         # If configuration type is not already set, try to read from attributes
 966         if vnf_cfg['interface_type'] is None:
 967             # Prepare unique name for this VNF
 968             vnf_unique_name = get_vnf_unique_name(
 969                     vnf_cfg['nsr_name'],
 970                     vnfr['short_name'],
 971                     vnfr['member_vnf_index_ref'],
 972                     )
 973
 974             # Find this particular (unique) VNF's config attributes
 975             if (vnf_unique_name in self.vnf_config_attributes_dict):
 976                 vnf_cfg_config_attributes_dict = self.vnf_config_attributes_dict[vnf_unique_name]
 977                 vnf_cfg['interface_type'] = vnf_cfg_config_attributes_dict['configuration_type']
 978                 if 'configuration_options' in vnf_cfg_config_attributes_dict:
 979                     cfg_opts = vnf_cfg_config_attributes_dict['configuration_options']
 980                     for key, value in cfg_opts.items():
 981                         vnf_cfg[key] = value
 982
 983         cfg_path_prefix = '{}/{}/{}_{}'.format(
 984                 self._parent._parent.cfg_dir,
 985                 vnf_cfg['nsr_name'],
 986                 vnfr['short_name'],
 987                 vnfr['member_vnf_index_ref'],
 988                 )
 989
 990         vnf_cfg['cfg_template'] = '{}_{}_template.cfg'.format(cfg_path_prefix, vnf_cfg['interface_type'])
 991         vnf_cfg['cfg_file'] = '{}.cfg'.format(cfg_path_prefix)
 992         vnf_cfg['xlate_script'] = self._parent._parent.cfg_dir + '/xlate_cfg.py'
 993
 994         self._log.debug("VNF endpoint so far: %s", vnf_cfg)
 995
 996         self._log.info("Checking cfg_template %s", vnf_cfg['cfg_template'])
 997         if os.path.exists(vnf_cfg['cfg_template']):
 998             return True
 999         return False
1000
1001     def ConfigVNF(self, vnfr):
1002
1003         vnf_cfg = vnfr['vnf_cfg']
1004         vnf_cm_state = self.find_or_create_vnfr_cm_state(vnf_cfg)
1005
1006         if (vnf_cm_state['state'] == self.state_to_string(conmanY.RecordState.READY_NO_CFG)
1007             or
1008             vnf_cm_state['state'] == self.state_to_string(conmanY.RecordState.READY)):
1009             self._log.warning("NS/VNF (%s/%s) is already configured! Skipped.", self.nsr_name, vnfr['short_name'])
1010             return
1011
1012         #UPdate VNF state
1013         vnf_cm_state['state'] = self.state_to_string(conmanY.RecordState.CFG_PROCESS)
1014
1015         # Now translate the configuration for iP addresses
1016         try:
1017             # Add cp_dict members (TAGS) for this VNF
1018             self._cp_dict['rw_mgmt_ip'] = vnf_cfg['mgmt_ip_address']
1019             self._cp_dict['rw_username'] = vnf_cfg['username']
1020             self._cp_dict['rw_password'] = vnf_cfg['password']
1021             ############################################################
1022             # TBD - Need to lookup above 3 for a given VNF, not global #
1023             # Once we do that no need to dump below file again before  #
1024             # each VNF configuration translation.                      #
1025             # This will require all existing config templates to be    #
1026             # changed for above three tags to include member index     #
1027             ############################################################
1028             try:
1029                 nsr_obj = vnf_cfg['nsr_obj']
1030                 # Generate config_config_attributes.yaml (For debug reference)
1031                 with open(nsr_obj.xlate_dict_file, "w") as yf:
1032                     yf.write(yaml.dump(nsr_obj._cp_dict, default_flow_style=False))
1033             except Exception as e:
1034                 self._log.error("NS:(%s) failed to write nsr xlate tags file as (%s)", nsr_obj.nsr_name, str(e))
1035
1036             if 'cfg_template' in vnf_cfg:
1037                 script_cmd = 'python3 {} -i {} -o {} -x "{}"'.format(vnf_cfg['xlate_script'], vnf_cfg['cfg_template'], vnf_cfg['cfg_file'], self.xlate_dict_file)
1038                 self._log.debug("xlate script command (%s)", script_cmd)
1039                 #xlate_msg = subprocess.check_output(script_cmd).decode('utf-8')
1040                 xlate_msg = subprocess.check_output(script_cmd, shell=True).decode('utf-8')
1041                 self._log.info("xlate script output (%s)", xlate_msg)
1042         except Exception as e:
1043             vnf_cm_state['state'] = self.state_to_string(conmanY.RecordState.CFG_PROCESS_FAILED)
1044             self._log.error("Failed to execute translation script for VNF: %s with (%s)", log_this_vnf(vnf_cfg), str(e))
1045             return
1046
1047         self._log.info("Applying config to VNF: %s = %s!", log_this_vnf(vnf_cfg), vnf_cfg)
1048         try:
1049             #self.vnf_cfg_list.append(vnf_cfg)
1050             self._log.debug("Scheduled configuration!")
1051             vnf_cm_state['state'] = self.state_to_string(conmanY.RecordState.CFG_SCHED)
1052         except Exception as e:
1053             self._log.error("Failed apply_vnf_config to VNF: %s as (%s)", log_this_vnf(vnf_cfg), str(e))
1054             vnf_cm_state['state'] = self.state_to_string(conmanY.RecordState.CFG_PROCESS_FAILED)
1055             raise
1056
1057     def add(self, nsr):
1058         self._log.info("Adding NS Record for id=%s", id)
1059         self._nsr = nsr
1060
1061     def sample_cm_state(self):
1062         return (
1063             {
1064                 'cm_nsr': [
1065                     {
1066                         'cm_vnfr': [
1067                             {
1068                                 'cfg_location': 'location1',
1069                                 'cfg_type': 'script',
1070                                 'connection_point': [
1071                                     {'ip_address': '1.1.1.1', 'name': 'vnf1cp1'},
1072                                     {'ip_address': '1.1.1.2', 'name': 'vnf1cp2'}
1073                                 ],
1074                                 'id': 'vnfrid1',
1075                                 'mgmt_interface': {'ip_address': '7.1.1.1',
1076                                                    'port': 1001},
1077                                 'name': 'vnfrname1',
1078                                 'state': 'init'
1079                             },
1080                             {
1081                                 'cfg_location': 'location2',
1082                                 'cfg_type': 'netconf',
1083                                 'connection_point': [{'ip_address': '2.1.1.1', 'name': 'vnf2cp1'},
1084                                                      {'ip_address': '2.1.1.2', 'name': 'vnf2cp2'}],
1085                                 'id': 'vnfrid2',
1086                                 'mgmt_interface': {'ip_address': '7.1.1.2',
1087                                                    'port': 1001},
1088                                 'name': 'vnfrname2',
1089                                 'state': 'init'}
1090                         ],
1091                         'id': 'nsrid1',
1092                         'name': 'nsrname1',
1093                         'state': 'init'}
1094                 ],
1095                 'states': 'Initialized, '
1096             })
1097
1098     def populate_vm_state_from_vnf_cfg(self):
1099         # Fill in each VNFR from this nsr object
1100         vnfr_list = self._vnfr_list
1101         for vnfr in vnfr_list:
1102             vnf_cfg = vnfr['vnf_cfg']
1103             vnf_cm_state = self.find_vnfr_cm_state(vnfr['id'])
1104
1105             if vnf_cm_state:
1106                 # Fill in VNF management interface
1107                 vnf_cm_state['mgmt_interface']['ip_address'] = vnf_cfg['mgmt_ip_address']
1108                 vnf_cm_state['mgmt_interface']['port'] = vnf_cfg['port']
1109
1110                 # Fill in VNF configuration details
1111                 vnf_cm_state['cfg_type'] = vnf_cfg['config_method']
1112                 vnf_cm_state['cfg_location'] = vnf_cfg['cfg_file']
1113
1114                 # Fill in each connection-point for this VNF
1115                 if "connection_point" in vnfr:
1116                     cp_list = vnfr['connection_point']
1117                     for cp_item_dict in cp_list:
1118                         vnf_cm_state['connection_point'].append(
1119                             {
1120                                 'name' : cp_item_dict['name'],
1121                                 'ip_address' : cp_item_dict['ip_address'],
1122                             }
1123                         )
1124
1125     def state_to_string(self, state):
1126         state_dict = {
1127             conmanY.RecordState.INIT : "init",
1128             conmanY.RecordState.RECEIVED : "received",
1129             conmanY.RecordState.CFG_PROCESS : "cfg_process",
1130             conmanY.RecordState.CFG_PROCESS_FAILED : "cfg_process_failed",
1131             conmanY.RecordState.CFG_SCHED : "cfg_sched",
1132             conmanY.RecordState.CFG_DELAY : "cfg_delay",
1133             conmanY.RecordState.CONNECTING : "connecting",
1134             conmanY.RecordState.FAILED_CONNECTION : "failed_connection",
1135             conmanY.RecordState.NETCONF_CONNECTED : "netconf_connected",
1136             conmanY.RecordState.NETCONF_SSH_CONNECTED : "netconf_ssh_connected",
1137             conmanY.RecordState.RESTCONF_CONNECTED : "restconf_connected",
1138             conmanY.RecordState.CFG_SEND : "cfg_send",
1139             conmanY.RecordState.CFG_FAILED : "cfg_failed",
1140             conmanY.RecordState.READY_NO_CFG : "ready_no_cfg",
1141             conmanY.RecordState.READY : "ready",
1142         }
1143         return state_dict[state]
1144
1145     def find_vnfr_cm_state(self, id):
1146         if self.cm_nsr['cm_vnfr']:
1147             for vnf_cm_state in self.cm_nsr['cm_vnfr']:
1148                 if vnf_cm_state['id'] == id:
1149                     return vnf_cm_state
1150         return None
1151
1152     def find_or_create_vnfr_cm_state(self, vnf_cfg):
1153         vnfr = vnf_cfg['vnfr']
1154         vnf_cm_state = self.find_vnfr_cm_state(vnfr['id'])
1155
1156         if vnf_cm_state is None:
1157             # Not found, Create and Initialize this VNF cm-state
1158             vnf_cm_state = {
1159                 'id' : vnfr['id'],
1160                 'name' : vnfr['short_name'],
1161                 'state' : self.state_to_string(conmanY.RecordState.RECEIVED),
1162                 'mgmt_interface' :
1163                 {
1164                     'ip_address' : vnf_cfg['mgmt_ip_address'],
1165                     'port' : vnf_cfg['port'],
1166                 },
1167                 'cfg_type' : vnf_cfg['config_method'],
1168                 'cfg_location' : vnf_cfg['cfg_file'],
1169                 'connection_point' : [],
1170             }
1171             self.cm_nsr['cm_vnfr'].append(vnf_cm_state)
1172
1173             # Publish newly created cm-state
1174
1175
1176         return vnf_cm_state
1177
1178     @asyncio.coroutine
1179     def get_vnf_cm_state(self, vnfr):
1180         if vnfr:
1181             vnf_cm_state = self.find_vnfr_cm_state(vnfr['id'])
1182             if vnf_cm_state:
1183                 return vnf_cm_state['state']
1184         return False
1185
1186     @asyncio.coroutine
1187     def update_vnf_cm_state(self, vnfr, state):
1188         if vnfr:
1189             vnf_cm_state = self.find_vnfr_cm_state(vnfr['id'])
1190             if vnf_cm_state is None:
1191                 self._log.error("No opdata found for NS/VNF:%s/%s!",
1192                                 self.nsr_name, vnfr['short_name'])
1193                 return
1194
1195             if vnf_cm_state['state'] != self.state_to_string(state):
1196                 old_state = vnf_cm_state['state']
1197                 vnf_cm_state['state'] = self.state_to_string(state)
1198                 # Publish new state
1199                 yield from self.publish_cm_state()
1200                 self._log.info("VNF ({}/{}/{}) state change: {} -> {}"
1201                                .format(self.nsr_name,
1202                                        vnfr['short_name'],
1203                                        vnfr['member_vnf_index_ref'],
1204                                        old_state,
1205                                        vnf_cm_state['state']))
1206
1207         else:
1208             self._log.error("No VNFR supplied for state update (NS=%s)!",
1209                             self.nsr_name)
1210
1211     @property
1212     def get_ns_cm_state(self):
1213         return self.cm_nsr['state']
1214
1215     @asyncio.coroutine
1216     def update_ns_cm_state(self, state, state_details=None):
1217         if self.cm_nsr['state'] != self.state_to_string(state):
1218             old_state = self.cm_nsr['state']
1219             self.cm_nsr['state'] = self.state_to_string(state)
1220             self.cm_nsr['state_details'] = state_details if state_details is not None else None
1221             self._log.info("NS ({}) state change: {} -> {}"
1222                            .format(self.nsr_name,
1223                                    old_state,
1224                                    self.cm_nsr['state']))
1225             # Publish new state
1226             yield from self.publish_cm_state()
1227
1228     @asyncio.coroutine
1229     def add_vnfr(self, vnfr, vnfr_msg):
1230
1231         @asyncio.coroutine
1232         def populate_subnets_from_vlr(id):
1233             try:
1234                 # Populate cp_dict with VLR subnet info
1235                 vlr = yield from self.dts_obj.get_vlr(id)
1236                 if vlr is not None and 'assigned_subnet' in vlr:
1237                     subnet = {vlr.name:vlr.assigned_subnet}
1238                     self._cp_dict[vnfr['member_vnf_index_ref']].update(subnet)
1239                     self._cp_dict.update(subnet)
1240                     self._log.debug("VNF:(%s) Updated assigned subnet = %s",
1241                                     vnfr['short_name'], subnet)
1242             except Exception as e:
1243                 self._log.error("VNF:(%s) VLR Error = %s",
1244                                 vnfr['short_name'], e)
1245
1246         if vnfr['id'] not in self._vnfr_dict:
1247             self._log.info("NSR(%s) : Adding VNF Record for name=%s, id=%s", self._nsr_id, vnfr['short_name'], vnfr['id'])
1248             # Add this vnfr to the list for show, or single traversal
1249             self._vnfr_list.append(vnfr)
1250         else:
1251             self._log.warning("NSR(%s) : VNF Record for name=%s, id=%s already exists, overwriting", self._nsr_id, vnfr['short_name'], vnfr['id'])
1252
1253         # Make vnfr available by id as well as by name
1254         unique_name = get_vnf_unique_name(self.nsr_name, vnfr['short_name'], vnfr['member_vnf_index_ref'])
1255         self._vnfr_dict[unique_name] = vnfr
1256         self._vnfr_dict[vnfr['id']] = vnfr
1257
1258         # Create vnf_cfg dictionary with default values
1259         vnf_cfg = {
1260             'nsr_obj' : self,
1261             'vnfr' : vnfr,
1262             'agent_vnfr' : self.agent_nsr.add_vnfr(vnfr, vnfr_msg),
1263             'nsr_name' : self.nsr_name,
1264             'nsr_id' : self._nsr_id,
1265             'vnfr_name' : vnfr['short_name'],
1266             'member_vnf_index' : vnfr['member_vnf_index_ref'],
1267             'port' : 0,
1268             'username' : 'admin',
1269             'password' : 'admin',
1270             'config_method' : 'None',
1271             'protocol' : 'None',
1272             'mgmt_ip_address' : '0.0.0.0',
1273             'cfg_file' : 'None',
1274             'cfg_retries' : 0,
1275             'script_type' : 'bash',
1276         }
1277
1278         # Update the mgmt ip address
1279         # In case the config method is none, this is not
1280         # updated later
1281         try:
1282             vnf_cfg['mgmt_ip_address'] = vnfr_msg.mgmt_interface.ip_address
1283             vnf_cfg['port'] = vnfr_msg.mgmt_interface.port
1284         except Exception as e:
1285             self._log.warn(
1286                 "VNFR {}({}), unable to retrieve mgmt ip address: {}".
1287                 format(vnfr['short_name'], vnfr['id'], e))
1288
1289         vnfr['vnf_cfg'] = vnf_cfg
1290         self.find_or_create_vnfr_cm_state(vnf_cfg)
1291
1292         '''
1293         Build the connection-points list for this VNF (self._cp_dict)
1294         '''
1295         # Populate global CP list self._cp_dict from VNFR
1296         cp_list = []
1297         if 'connection_point' in vnfr:
1298             cp_list = vnfr['connection_point']
1299
1300         self._cp_dict[vnfr['member_vnf_index_ref']] = {}
1301         if 'vdur' in vnfr:
1302             for vdur in vnfr['vdur']:
1303                 if 'internal_connection_point' in vdur:
1304                     cp_list += vdur['internal_connection_point']
1305
1306                 for cp_item_dict in cp_list:
1307                     # Populate global dictionary
1308                     self._cp_dict[
1309                         cp_item_dict['name']
1310                     ] = cp_item_dict['ip_address']
1311
1312                     # Populate unique member specific dictionary
1313                     self._cp_dict[
1314                         vnfr['member_vnf_index_ref']
1315                     ][
1316                         cp_item_dict['name']
1317                     ] = cp_item_dict['ip_address']
1318
1319                     # Fill in the subnets from vlr
1320                     if 'vlr_ref' in cp_item_dict:
1321                         ### HACK: Internal connection_point do not have VLR reference
1322                         yield from populate_subnets_from_vlr(cp_item_dict['vlr_ref'])
1323
1324         if 'internal_vlr' in vnfr:
1325             for ivlr in vnfr['internal_vlr']:
1326                 yield from populate_subnets_from_vlr(ivlr['vlr_ref'])
1327
1328         # Update vnfr
1329         vnf_cfg['agent_vnfr']._vnfr = vnfr
1330         return vnf_cfg['agent_vnfr']
1331
1332
1333 class XPaths(object):
1334     @staticmethod
1335     def nsr_opdata(k=None):
1336         return ("D,/nsr:ns-instance-opdata/nsr:nsr" +
1337                 ("[nsr:ns-instance-config-ref='{}']".format(k) if k is not None else ""))
1338
1339     @staticmethod
1340     def nsd_msg(k=None):
1341         return ("C,/nsd:nsd-catalog/nsd:nsd" +
1342                 "[nsd:id = '{}']".format(k) if k is not None else "")
1343
1344     @staticmethod
1345     def vnfr_opdata(k=None):
1346         return ("D,/vnfr:vnfr-catalog/vnfr:vnfr" +
1347                 ("[vnfr:id='{}']".format(k) if k is not None else ""))
1348
1349     @staticmethod
1350     def vnfd(k=None):
1351         return ("C,/vnfd:vnfd-catalog/vnfd:vnfd" +
1352                 ("[vnfd:id='{}']".format(k) if k is not None else ""))
1353
1354     @staticmethod
1355     def config_agent(k=None):
1356         return ("D,/rw-config-agent:config-agent/rw-config-agent:account" +
1357                 ("[rw-config-agent:name='{}']".format(k) if k is not None else ""))
1358
1359     @staticmethod
1360     def nsr_config(k=None):
1361         return ("C,/nsr:ns-instance-config/nsr:nsr[nsr:id='{}']".format(k) if k is not None else "")
1362
1363     @staticmethod
1364     def vlr(k=None):
1365         return ("D,/vlr:vlr-catalog/vlr:vlr[vlr:id='{}']".format(k) if k is not None else "")
1366
1367 class ConfigManagerDTS(object):
1368     ''' This class either reads from DTS or publishes to DTS '''
1369
1370     def __init__(self, log, loop, parent, dts):
1371         self._log = log
1372         self._loop = loop
1373         self._parent = parent
1374         self._dts = dts
1375
1376     @asyncio.coroutine
1377     def _read_dts(self, xpath, do_trace=False):
1378         self._log.debug("_read_dts path = %s", xpath)
1379         flags = rwdts.XactFlag.MERGE
1380         res_iter = yield from self._dts.query_read(
1381                 xpath, flags=flags
1382                 )
1383
1384         results = []
1385         try:
1386             for i in res_iter:
1387                 result = yield from i
1388                 if result is not None:
1389                     results.append(result.result)
1390         except:
1391             pass
1392
1393         return results
1394
1395
1396     @asyncio.coroutine
1397     def get_nsr(self, id):
1398         self._log.debug("Attempting to get NSR: %s", id)
1399         nsrl = yield from self._read_dts(XPaths.nsr_opdata(id), False)
1400         nsr = None
1401         if len(nsrl) > 0:
1402             nsr =  nsrl[0].as_dict()
1403         return nsr
1404
1405     @asyncio.coroutine
1406     def get_nsr_config(self, id):
1407         self._log.debug("Attempting to get config NSR: %s", id)
1408         nsrl = yield from self._read_dts(XPaths.nsr_config(id), False)
1409         nsr = None
1410         if len(nsrl) > 0:
1411             nsr =  nsrl[0]
1412         return nsr
1413
1414     @asyncio.coroutine
1415     def get_nsd_msg(self, id):
1416         self._log.debug("Attempting to get NSD: %s", id)
1417         nsdl = yield from self._read_dts(XPaths.nsd_msg(id), False)
1418         nsd_msg = None
1419         if len(nsdl) > 0:
1420             nsd_msg =  nsdl[0]
1421         return nsd_msg
1422
1423     @asyncio.coroutine
1424     def get_nsd(self, nsr_id):
1425         self._log.debug("Attempting to get NSD for NSR: %s", id)
1426         nsr_config = yield from self.get_nsr_config(nsr_id)
1427         nsd_msg = nsr_config.nsd
1428         return nsd_msg
1429
1430     @asyncio.coroutine
1431     def get_vnfr(self, id):
1432         self._log.debug("Attempting to get VNFR: %s", id)
1433         vnfrl = yield from self._read_dts(XPaths.vnfr_opdata(id), do_trace=False)
1434         vnfr_msg = None
1435         if len(vnfrl) > 0:
1436             vnfr_msg = vnfrl[0]
1437         return vnfr_msg
1438
1439     @asyncio.coroutine
1440     def get_vnfd(self, vnfd_id):
1441         self._log.debug("Attempting to get VNFD: %s", vnfd_id)
1442         vnfdl = yield from self._read_dts(XPaths.vnfd(vnfd_id), do_trace=False)
1443         vnfd_msg = None
1444         if len(vnfdl) > 0:
1445             vnfd_msg = vnfdl[0]
1446         return vnfd_msg
1447
1448     @asyncio.coroutine
1449     def get_vlr(self, id):
1450         self._log.debug("Attempting to get VLR subnet: %s", id)
1451         vlrl = yield from self._read_dts(XPaths.vlr(id), do_trace=True)
1452         vlr_msg = None
1453         if len(vlrl) > 0:
1454             vlr_msg = vlrl[0]
1455         return vlr_msg
1456
1457     @asyncio.coroutine
1458     def get_config_agents(self, name):
1459         self._log.debug("Attempting to get config_agents: %s", name)
1460         cfgagentl = yield from self._read_dts(XPaths.config_agent(name), False)
1461         return cfgagentl
1462
1463     @asyncio.coroutine
1464     def update(self, path, msg, flags=rwdts.XactFlag.REPLACE):
1465         """
1466         Update a cm-state (cm-nsr) record in DTS with the path and message
1467         """
1468         self._log.debug("Updating cm-state %s:%s dts_pub_hdl = %s", path, msg, self.dts_pub_hdl)
1469         self.dts_pub_hdl.update_element(path, msg, flags)
1470         self._log.debug("Updated cm-state, %s:%s", path, msg)
1471
1472     @asyncio.coroutine
1473     def delete(self, path):
1474         """
1475         Delete cm-nsr record in DTS with the path only
1476         """
1477         self._log.debug("Deleting cm-nsr %s dts_pub_hdl = %s", path, self.dts_pub_hdl)
1478         self.dts_pub_hdl.delete_element(path)
1479         self._log.debug("Deleted cm-nsr, %s", path)
1480
1481     @asyncio.coroutine
1482     def register(self):
1483         yield from self.register_to_publish()
1484         yield from self.register_for_nsr()
1485
1486     @asyncio.coroutine
1487     def register_to_publish(self):
1488         ''' Register to DTS for publishing cm-state opdata '''
1489
1490         xpath = "D,/rw-conman:cm-state/rw-conman:cm-nsr"
1491         self._log.debug("Registering to publish cm-state @ %s", xpath)
1492         hdl = rift.tasklets.DTS.RegistrationHandler()
1493         with self._dts.group_create() as group:
1494             self.dts_pub_hdl = group.register(xpath=xpath,
1495                                               handler=hdl,
1496                                               flags=rwdts.Flag.PUBLISHER | rwdts.Flag.NO_PREP_READ)
1497
1498     @property
1499     def nsr_xpath(self):
1500         return "D,/nsr:ns-instance-opdata/nsr:nsr"
1501
1502     @asyncio.coroutine
1503     def register_for_nsr(self):
1504         """ Register for NSR changes """
1505
1506         @asyncio.coroutine
1507         def on_prepare(xact_info, query_action, ks_path, msg):
1508             """ This NSR is created """
1509             self._log.debug("Received NSR instantiate on_prepare (%s:%s:%s)",
1510                             query_action,
1511                             ks_path,
1512                             msg)
1513
1514             if (query_action == rwdts.QueryAction.UPDATE or
1515                 query_action == rwdts.QueryAction.CREATE):
1516                 msg_dict = msg.as_dict()
1517                 # Update Each NSR/VNFR state)
1518                 if ('operational_status' in msg_dict and
1519                     msg_dict['operational_status'] == 'running'):
1520                     # Add to the task list
1521                     self._parent.add_to_pending_tasks({'nsrid' : msg_dict['ns_instance_config_ref'], 'retries' : 5})
1522             elif query_action == rwdts.QueryAction.DELETE:
1523                 nsr_id = msg.ns_instance_config_ref
1524                 asyncio.ensure_future(self._parent.terminate_NSR(nsr_id), loop=self._loop)
1525             else:
1526                 raise NotImplementedError(
1527                     "%s action on cm-state not supported",
1528                     query_action)
1529
1530             xact_info.respond_xpath(rwdts.XactRspCode.ACK)
1531
1532         try:
1533             handler = rift.tasklets.DTS.RegistrationHandler(on_prepare=on_prepare)
1534             self.dts_reg_hdl = yield from self._dts.register(self.nsr_xpath,
1535                                                              flags=rwdts.Flag.SUBSCRIBER | rwdts.Flag.DELTA_READY,
1536                                                              handler=handler)
1537         except Exception as e:
1538             self._log.error("Failed to register for NSR changes as %s", str(e))
1539
1540