rwcm/plugins/rwconman/rift/tasklets/rwconmantasklet/rwconman_config.py

   1
   2 #
   3 #   Copyright 2016 RIFT.IO Inc
   4 #
   5 #   Licensed under the Apache License, Version 2.0 (the "License");
   6 #   you may not use this file except in compliance with the License.
   7 #   You may obtain a copy of the License at
   8 #
   9 #       http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 #   Unless required by applicable law or agreed to in writing, software
  12 #   distributed under the License is distributed on an "AS IS" BASIS,
  13 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 #   See the License for the specific language governing permissions and
  15 #   limitations under the License.
  16 #
  17
  18 import asyncio
  19 import os
  20 import stat
  21 import subprocess
  22 import sys
  23 import tempfile
  24 import yaml
  25
  26 from gi.repository import (
  27     RwDts as rwdts,
  28     RwConmanYang as conmanY,
  29     ProtobufC,
  30 )
  31
  32 import rift.tasklets
  33
  34 from . import rwconman_conagent as conagent
  35 from . import RiftCM_rpc
  36 from . import riftcm_config_plugin
  37
  38 if sys.version_info < (3, 4, 4):
  39     asyncio.ensure_future = asyncio.async
  40
  41 def get_vnf_unique_name(nsr_name, vnfr_short_name, member_vnf_index):
  42     return "{}.{}.{}".format(nsr_name, vnfr_short_name, member_vnf_index)
  43
  44 class ConmanConfigError(Exception):
  45     pass
  46
  47
  48 class InitialConfigError(ConmanConfigError):
  49     pass
  50
  51
  52 def log_this_vnf(vnf_cfg):
  53     log_vnf = ""
  54     used_item_list = ['nsr_name', 'vnfr_name', 'member_vnf_index', 'mgmt_ip_address']
  55     for item in used_item_list:
  56         if item in vnf_cfg:
  57             if item == 'mgmt_ip_address':
  58                 log_vnf += "({})".format(vnf_cfg[item])
  59             else:
  60                 log_vnf += "{}/".format(vnf_cfg[item])
  61     return log_vnf
  62
  63 class PretendNsm(object):
  64     def __init__(self, dts, log, loop, parent):
  65         self._dts = dts
  66         self._log = log
  67         self._loop = loop
  68         self._parent = parent
  69         self._nsrs = {}
  70         self._nsr_dict = parent._nsr_dict
  71         self._config_agent_plugins = []
  72         self._nsd_msg = {}
  73
  74     @property
  75     def nsrs(self):
  76         # Expensive, instead use get_nsr, if you know id.
  77         self._nsrs = {}
  78         # Update the list of nsrs (agent nsr)
  79         for id, nsr_obj in self._nsr_dict.items():
  80             self._nsrs[id] = nsr_obj.agent_nsr
  81         return self._nsrs
  82
  83     def get_nsr(self, nsr_id):
  84         if nsr_id in self._nsr_dict:
  85             nsr_obj = self._nsr_dict[nsr_id]
  86             return nsr_obj._nsr
  87         return None
  88
  89     def get_vnfr_msg(self, vnfr_id, nsr_id=None):
  90         self._log.debug("get_vnfr_msg(vnfr=%s, nsr=%s)",
  91                         vnfr_id, nsr_id)
  92         found = False
  93         if nsr_id:
  94             if nsr_id in self._nsr_dict:
  95                 nsr_obj = self._nsr_dict[nsr_id]
  96                 if vnfr_id in nsr_obj._vnfr_dict:
  97                     found = True
  98         else:
  99             for nsr_obj in self._nsr_dict.values():
 100                 if vnfr_id in nsr_obj._vnfr_dict:
 101                     # Found it
 102                     found = True
 103                     break
 104         if found:
 105             vnf_cfg = nsr_obj._vnfr_dict[vnfr_id]['vnf_cfg']
 106             return vnf_cfg['agent_vnfr'].vnfr_msg
 107         else:
 108             return None
 109
 110     @asyncio.coroutine
 111     def get_nsd(self, nsr_id):
 112         if nsr_id not in self._nsd_msg:
 113             nsr_config = yield from self._parent.cmdts_obj.get_nsr_config(nsr_id)
 114             self._nsd_msg[nsr_id] = nsr_config.nsd
 115         return self._nsd_msg[nsr_id]
 116
 117     @property
 118     def config_agent_plugins(self):
 119         self._config_agent_plugins = []
 120         for agent in self._parent._config_agent_mgr._plugin_instances.values():
 121             self._config_agent_plugins.append(agent)
 122         return self._config_agent_plugins
 123
 124 class ConfigManagerConfig(object):
 125     def __init__(self, dts, log, loop, parent):
 126         self._dts = dts
 127         self._log = log
 128         self._loop = loop
 129         self._parent = parent
 130         self._nsr_dict = {}
 131         self.pending_cfg = {}
 132         self.terminate_cfg = {}
 133         self.pending_tasks = [] # User for NSRid get retry
 134                                 # (mainly excercised at restart case)
 135         self._config_xpath = "C,/cm-config"
 136         self._opdata_xpath = "D,/rw-conman:cm-state"
 137
 138         self.cm_config = conmanY.SoConfig()
 139         # RO specific configuration
 140         self.ro_config = {}
 141         for key in self.cm_config.ro_endpoint.fields:
 142             self.ro_config[key] = None
 143
 144         # Initialize cm-state
 145         self.cm_state = {}
 146         self.cm_state['cm_nsr'] = []
 147         self.cm_state['states'] = "Initialized"
 148
 149         # Initialize objects to register
 150         self.cmdts_obj = ConfigManagerDTS(self._log, self._loop, self, self._dts)
 151         self._config_agent_mgr = conagent.RiftCMConfigAgent(
 152             self._dts,
 153             self._log,
 154             self._loop,
 155             self,
 156         )
 157         self.reg_handles = [
 158             self.cmdts_obj,
 159             self._config_agent_mgr,
 160             RiftCM_rpc.RiftCMRPCHandler(self._dts, self._log, self._loop,
 161                                         PretendNsm(
 162                                             self._dts, self._log, self._loop, self)),
 163         ]
 164
 165     def is_nsr_valid(self, nsr_id):
 166         if nsr_id in self._nsr_dict:
 167             return True
 168         return False
 169
 170     def add_to_pending_tasks(self, task):
 171         if self.pending_tasks:
 172             for p_task in self.pending_tasks:
 173                 if p_task['nsrid'] == task['nsrid']:
 174                     # Already queued
 175                     return
 176         try:
 177             self.pending_tasks.append(task)
 178             self._log.debug("add_to_pending_tasks (nsrid:%s)",
 179                             task['nsrid'])
 180             if len(self.pending_tasks) == 1:
 181                 self._loop.create_task(self.ConfigManagerConfig_pending_loop())
 182                 # TBD - change to info level
 183                 self._log.debug("Started pending_loop!")
 184         except Exception as e:
 185             self._log.error("Failed adding to pending tasks (%s)", str(e))
 186
 187     def del_from_pending_tasks(self, task):
 188         try:
 189             self.pending_tasks.remove(task)
 190         except Exception as e:
 191             self._log.error("Failed removing from pending tasks (%s)", str(e))
 192
 193     @asyncio.coroutine
 194     def ConfigManagerConfig_pending_loop(self):
 195         loop_sleep = 2
 196         while True:
 197             yield from asyncio.sleep(loop_sleep, loop=self._loop)
 198             """
 199             This pending task queue is ordred by events,
 200             must finish previous task successfully to be able to go on to the next task
 201             """
 202             if self.pending_tasks:
 203                 self._log.debug("self.pending_tasks len=%s", len(self.pending_tasks))
 204                 task = self.pending_tasks[0]
 205                 done = False
 206                 if 'nsrid' in task:
 207                     nsrid = task['nsrid']
 208                     self._log.debug("Will execute pending task for NSR id(%s)", nsrid)
 209                     try:
 210                         # Try to configure this NSR
 211                         task['retries'] -= 1
 212                         done = yield from self.config_NSR(nsrid)
 213                         self._log.info("self.config_NSR status=%s", done)
 214
 215                     except Exception as e:
 216                         self._log.error("Failed(%s) configuring NSR(%s)," \
 217                                         "retries remained:%d!",
 218                                         str(e), nsrid, task['retries'])
 219                     finally:
 220                         self.pending_tasks.remove(task)
 221
 222                     if done:
 223                         self._log.debug("Finished pending task NSR id(%s):", nsrid)
 224                     else:
 225                         self._log.error("Failed configuring NSR(%s), retries remained:%d!",
 226                                         nsrid, task['retries'])
 227
 228                         # Failed, re-insert (append at the end)
 229                         # this failed task to be retried later
 230                         # If any retries remained.
 231                         if task['retries']:
 232                             self.pending_tasks.append(task)
 233             else:
 234                 self._log.debug("Stopped pending_loop!")
 235                 break
 236
 237     @asyncio.coroutine
 238     def register(self):
 239         yield from self.register_cm_state_opdata()
 240
 241         # Initialize all handles that needs to be registered
 242         for reg in self.reg_handles:
 243             yield from reg.register()
 244
 245     @asyncio.coroutine
 246     def register_cm_state_opdata(self):
 247
 248         def state_to_string(state):
 249             state_dict = {
 250                 conmanY.RecordState.INIT : "init",
 251                 conmanY.RecordState.RECEIVED : "received",
 252                 conmanY.RecordState.CFG_PROCESS : "cfg_process",
 253                 conmanY.RecordState.CFG_PROCESS_FAILED : "cfg_process_failed",
 254                 conmanY.RecordState.CFG_SCHED : "cfg_sched",
 255                 conmanY.RecordState.CFG_DELAY : "cfg_delay",
 256                 conmanY.RecordState.CONNECTING : "connecting",
 257                 conmanY.RecordState.FAILED_CONNECTION : "failed_connection",
 258                 conmanY.RecordState.NETCONF_CONNECTED : "netconf_connected",
 259                 conmanY.RecordState.NETCONF_SSH_CONNECTED : "netconf_ssh_connected",
 260                 conmanY.RecordState.RESTCONF_CONNECTED : "restconf_connected",
 261                 conmanY.RecordState.CFG_SEND : "cfg_send",
 262                 conmanY.RecordState.CFG_FAILED : "cfg_failed",
 263                 conmanY.RecordState.READY_NO_CFG : "ready_no_cfg",
 264                 conmanY.RecordState.READY : "ready",
 265                 }
 266             return state_dict[state]
 267
 268         @asyncio.coroutine
 269         def on_prepare(xact_info, action, ks_path, msg):
 270
 271             self._log.debug("Received cm-state: msg=%s, action=%s", msg, action)
 272
 273             if action == rwdts.QueryAction.READ:
 274                 show_output = conmanY.CmOpdata()
 275                 show_output.from_dict(self.cm_state)
 276                 self._log.debug("Responding to SHOW cm-state: %s", self.cm_state)
 277                 xact_info.respond_xpath(rwdts.XactRspCode.ACK,
 278                                         xpath=self._opdata_xpath,
 279                                         msg=show_output)
 280             else:
 281                 xact_info.respond_xpath(rwdts.XactRspCode.ACK)
 282
 283         self._log.info("Registering for cm-opdata xpath: %s",
 284                         self._opdata_xpath)
 285
 286         try:
 287             handler=rift.tasklets.DTS.RegistrationHandler(on_prepare=on_prepare)
 288             yield from self._dts.register(xpath=self._opdata_xpath,
 289                                           handler=handler,
 290                                           flags=rwdts.Flag.PUBLISHER)
 291             self._log.info("Successfully registered for opdata(%s)", self._opdata_xpath)
 292         except Exception as e:
 293             self._log.error("Failed to register for opdata as (%s)", e)
 294
 295     @asyncio.coroutine
 296     def process_nsd_vnf_configuration(self, nsr_obj, vnfr):
 297
 298         def get_config_method(vnf_config):
 299             cfg_types = ['netconf', 'juju', 'script']
 300             for method in cfg_types:
 301                 if method in vnf_config:
 302                     return method
 303             return None
 304
 305         def get_cfg_file_extension(method,  configuration_options):
 306             ext_dict = {
 307                 "netconf" : "xml",
 308                 "script" : {
 309                     "bash" : "sh",
 310                     "expect" : "exp",
 311                 },
 312                 "juju" : "yml"
 313             }
 314
 315             if method == "netconf":
 316                 return ext_dict[method]
 317             elif method == "script":
 318                 return ext_dict[method][configuration_options['script_type']]
 319             elif method == "juju":
 320                 return ext_dict[method]
 321             else:
 322                 return "cfg"
 323
 324         # This is how the YAML file should look like,
 325         # This routine will be called for each VNF, so keep appending the file.
 326         # priority order is determined by the number,
 327         # hence no need to generate the file in that order. A dictionary will be
 328         # used that will take care of the order by number.
 329         '''
 330         1 : <== This is priority
 331           name : trafsink_vnfd
 332           member_vnf_index : 2
 333           configuration_delay : 120
 334           configuration_type : netconf
 335           configuration_options :
 336             username : admin
 337             password : admin
 338             port : 2022
 339             target : running
 340         2 :
 341           name : trafgen_vnfd
 342           member_vnf_index : 1
 343           configuration_delay : 0
 344           configuration_type : netconf
 345           configuration_options :
 346             username : admin
 347             password : admin
 348             port : 2022
 349             target : running
 350         '''
 351
 352         # Save some parameters needed as short cuts in flat structure (Also generated)
 353         vnf_cfg = vnfr['vnf_cfg']
 354         # Prepare unique name for this VNF
 355         vnf_cfg['vnf_unique_name'] = get_vnf_unique_name(
 356             vnf_cfg['nsr_name'], vnfr['short_name'], vnfr['member_vnf_index_ref'])
 357
 358         nsr_obj.cfg_path_prefix = '{}/{}_{}'.format(
 359             nsr_obj.this_nsr_dir, vnfr['short_name'], vnfr['member_vnf_index_ref'])
 360         nsr_vnfr = '{}/{}_{}'.format(
 361             vnf_cfg['nsr_name'], vnfr['short_name'], vnfr['member_vnf_index_ref'])
 362
 363         # Get vnf_configuration from vnfr
 364         vnf_config = vnfr['vnf_configuration']
 365
 366         self._log.debug("vnf_configuration = %s", vnf_config)
 367
 368         # Create priority dictionary
 369         cfg_priority_order = 0
 370         if ('config_attributes' in vnf_config and
 371             'config_priority' in vnf_config['config_attributes']):
 372             cfg_priority_order = vnf_config['config_attributes']['config_priority']
 373
 374         if cfg_priority_order not in nsr_obj.nsr_cfg_config_attributes_dict:
 375             # No VNFR with this priority yet, initialize the list
 376             nsr_obj.nsr_cfg_config_attributes_dict[cfg_priority_order] = []
 377
 378         method = get_config_method(vnf_config)
 379         if method is not None:
 380             # Create all sub dictionaries first
 381             config_priority = {
 382                 'id' : vnfr['id'],
 383                 'name' : vnfr['short_name'],
 384                 'member_vnf_index' : vnfr['member_vnf_index_ref'],
 385             }
 386
 387             if 'config_delay' in vnf_config['config_attributes']:
 388                 config_priority['configuration_delay'] = vnf_config['config_attributes']['config_delay']
 389                 vnf_cfg['config_delay'] = config_priority['configuration_delay']
 390
 391             configuration_options = {}
 392             self._log.debug("config method=%s", method)
 393             config_priority['configuration_type'] = method
 394             vnf_cfg['config_method'] = method
 395
 396             # Set config agent based on method
 397             self._config_agent_mgr.set_config_agent(
 398                 nsr_obj.agent_nsr, vnf_cfg['agent_vnfr'], method)
 399
 400             cfg_opt_list = [
 401                 'port', 'target', 'script_type', 'ip_address', 'user', 'secret',
 402             ]
 403             for cfg_opt in cfg_opt_list:
 404                 if cfg_opt in vnf_config[method]:
 405                     configuration_options[cfg_opt] = vnf_config[method][cfg_opt]
 406                     vnf_cfg[cfg_opt] = configuration_options[cfg_opt]
 407
 408             cfg_opt_list = ['mgmt_ip_address', 'username', 'password']
 409             for cfg_opt in cfg_opt_list:
 410                 if cfg_opt in vnf_config['config_access']:
 411                     configuration_options[cfg_opt] = vnf_config['config_access'][cfg_opt]
 412                     vnf_cfg[cfg_opt] = configuration_options[cfg_opt]
 413
 414             # Add to the cp_dict
 415             vnf_cp_dict = nsr_obj._cp_dict[vnfr['member_vnf_index_ref']]
 416             vnf_cp_dict['rw_mgmt_ip'] = vnf_cfg['mgmt_ip_address']
 417             vnf_cp_dict['rw_username'] = vnf_cfg['username']
 418             vnf_cp_dict['rw_password'] = vnf_cfg['password']
 419
 420
 421             # TBD - see if we can neatly include the config in "config_attributes" file, no need though
 422             #config_priority['config_template'] = vnf_config['config_template']
 423             # Create config file
 424             vnf_cfg['juju_script'] = os.path.join(self._parent.cfg_dir, 'juju_if.py')
 425
 426             if 'config_template' in vnf_config:
 427                 vnf_cfg['cfg_template'] = '{}_{}_template.cfg'.format(nsr_obj.cfg_path_prefix, config_priority['configuration_type'])
 428                 vnf_cfg['cfg_file'] = '{}.{}'.format(nsr_obj.cfg_path_prefix, get_cfg_file_extension(method, configuration_options))
 429                 vnf_cfg['xlate_script'] = os.path.join(self._parent.cfg_dir, 'xlate_cfg.py')
 430                 try:
 431                     # Now write this template into file
 432                     with open(vnf_cfg['cfg_template'], "w") as cf:
 433                         cf.write(vnf_config['config_template'])
 434                 except Exception as e:
 435                     self._log.error("Processing NSD, failed to generate configuration template : %s (Error : %s)",
 436                                     vnf_config['config_template'], str(e))
 437                     raise
 438
 439             self._log.debug("VNF endpoint so far: %s", vnf_cfg)
 440
 441             # Populate filled up dictionary
 442             config_priority['configuration_options'] = configuration_options
 443             nsr_obj.nsr_cfg_config_attributes_dict[cfg_priority_order].append(config_priority)
 444             nsr_obj.num_vnfs_to_cfg += 1
 445             nsr_obj._vnfr_dict[vnf_cfg['vnf_unique_name']] = vnfr
 446             nsr_obj._vnfr_dict[vnfr['id']] = vnfr
 447
 448             self._log.debug("VNF:(%s) config_attributes = %s",
 449                             log_this_vnf(vnfr['vnf_cfg']),
 450                             nsr_obj.nsr_cfg_config_attributes_dict)
 451         else:
 452             self._log.info("VNF:(%s) is not to be configured by Configuration Manager!",
 453                            log_this_vnf(vnfr['vnf_cfg']))
 454             yield from nsr_obj.update_vnf_cm_state(vnfr, conmanY.RecordState.READY_NO_CFG)
 455
 456         # Update the cm-state
 457         nsr_obj.populate_vm_state_from_vnf_cfg()
 458
 459     @asyncio.coroutine
 460     def config_NSR(self, id):
 461
 462         def my_yaml_dump(config_attributes_dict, yf):
 463
 464             yaml_dict = dict(sorted(config_attributes_dict.items()))
 465             yf.write(yaml.dump(yaml_dict, default_flow_style=False))
 466
 467         nsr_dict = self._nsr_dict
 468         self._log.info("Configure NSR, id = %s", id)
 469
 470         #####################TBD###########################
 471         # yield from self._config_agent_mgr.invoke_config_agent_plugins('notify_create_nsr', self.id, self._nsd)
 472         # yield from self._config_agent_mgr.invoke_config_agent_plugins('notify_nsr_active', self.id, self._vnfrs)
 473
 474         try:
 475             if id not in nsr_dict:
 476                 nsr_obj = ConfigManagerNSR(self._log, self._loop, self, id)
 477                 nsr_dict[id] = nsr_obj
 478             else:
 479                 self._log.info("NSR(%s) is already initialized!", id)
 480                 nsr_obj = nsr_dict[id]
 481         except Exception as e:
 482             self._log.error("Failed creating NSR object for (%s) as (%s)", id, str(e))
 483             raise
 484
 485         # Try to configure this NSR only if not already processed
 486         if nsr_obj.cm_nsr['state'] != nsr_obj.state_to_string(conmanY.RecordState.INIT):
 487             self._log.debug("NSR(%s) is already processed, state=%s",
 488                             nsr_obj.nsr_name, nsr_obj.cm_nsr['state'])
 489             yield from nsr_obj.publish_cm_state()
 490             return True
 491
 492         cmdts_obj = self.cmdts_obj
 493         try:
 494             # Fetch NSR
 495             nsr = yield from cmdts_obj.get_nsr(id)
 496             self._log.debug("Full NSR : %s", nsr)
 497             if nsr['operational_status'] != "running":
 498                 self._log.info("NSR(%s) is not ready yet!", nsr['nsd_name_ref'])
 499                 return False
 500             self._nsr = nsr
 501
 502             # Create Agent NSR class
 503             nsr_config = yield from cmdts_obj.get_nsr_config(id)
 504             self._log.debug("NSR {} config: {}".format(id, nsr_config))
 505             nsr_obj.agent_nsr = riftcm_config_plugin.RiftCMnsr(nsr, nsr_config)
 506
 507             try:
 508                 yield from nsr_obj.update_ns_cm_state(conmanY.RecordState.RECEIVED)
 509
 510                 # Parse NSR
 511                 if nsr is not None:
 512                     nsr_obj.set_nsr_name(nsr['name_ref'])
 513                     nsr_dir = os.path.join(self._parent.cfg_dir, nsr_obj.nsr_name)
 514                     self._log.info("Checking NS config directory: %s", nsr_dir)
 515                     if not os.path.isdir(nsr_dir):
 516                         os.makedirs(nsr_dir)
 517                         # self._log.critical("NS %s is not to be configured by Service Orchestrator!", nsr_obj.nsr_name)
 518                         # yield from nsr_obj.update_ns_cm_state(conmanY.RecordState.READY_NO_CFG)
 519                         # return
 520
 521                     nsr_obj.set_config_dir(self)
 522
 523                     for const_vnfr in nsr['constituent_vnfr_ref']:
 524                         self._log.debug("Fetching VNFR (%s)", const_vnfr['vnfr_id'])
 525                         vnfr_msg = yield from cmdts_obj.get_vnfr(const_vnfr['vnfr_id'])
 526                         if vnfr_msg:
 527                             vnfr = vnfr_msg.as_dict()
 528                             self._log.info("create VNF:{}/{}".format(nsr_obj.nsr_name, vnfr['short_name']))
 529                             agent_vnfr = yield from nsr_obj.add_vnfr(vnfr, vnfr_msg)
 530
 531                             # Preserve order, self.process_nsd_vnf_configuration()
 532                             # sets up the config agent based on the method
 533                             yield from self.process_nsd_vnf_configuration(nsr_obj, vnfr)
 534                             yield from self._config_agent_mgr.invoke_config_agent_plugins(
 535                                 'notify_create_vnfr',
 536                                 nsr_obj.agent_nsr,
 537                                 agent_vnfr)
 538
 539                         #####################TBD###########################
 540                         # self._log.debug("VNF active. Apply initial config for vnfr {}".format(vnfr.name))
 541                         # yield from self._config_agent_mgr.invoke_config_agent_plugins('apply_initial_config',
 542                         #                                             vnfr.id, vnfr)
 543                         # yield from self._config_agent_mgr.invoke_config_agent_plugins('notify_terminate_vnf', self.id, vnfr)
 544
 545             except Exception as e:
 546                 self._log.error("Failed processing NSR (%s) as (%s)", nsr_obj.nsr_name, str(e))
 547                 self._log.exception(e)
 548                 yield from nsr_obj.update_ns_cm_state(conmanY.RecordState.CFG_PROCESS_FAILED)
 549                 raise
 550
 551             try:
 552                 # Generate config_config_attributes.yaml (For debug reference)
 553                 with open(nsr_obj.config_attributes_file, "w") as yf:
 554                     my_yaml_dump(nsr_obj.nsr_cfg_config_attributes_dict, yf)
 555             except Exception as e:
 556                 self._log.error("NS:(%s) failed to write config attributes file as (%s)", nsr_obj.nsr_name, str(e))
 557
 558             try:
 559                 # Generate nsr_xlate_dict.yaml (For debug reference)
 560                 with open(nsr_obj.xlate_dict_file, "w") as yf:
 561                     yf.write(yaml.dump(nsr_obj._cp_dict, default_flow_style=False))
 562             except Exception as e:
 563                 self._log.error("NS:(%s) failed to write nsr xlate tags file as (%s)", nsr_obj.nsr_name, str(e))
 564
 565             self._log.debug("Starting to configure each VNF")
 566
 567             # Check if this NS has input parametrs
 568             self._log.info("Checking NS configuration order: %s", nsr_obj.config_attributes_file)
 569
 570             if os.path.exists(nsr_obj.config_attributes_file):
 571                 # Apply configuration is specified order
 572                 try:
 573                     # Go in loop to configure by specified order
 574                     self._log.info("Using Dynamic configuration input parametrs for NS: %s", nsr_obj.nsr_name)
 575
 576                     # cfg_delay = nsr_obj.nsr_cfg_config_attributes_dict['configuration_delay']
 577                     # if cfg_delay:
 578                     #     self._log.info("Applying configuration delay for NS (%s) ; %d seconds",
 579                     #                    nsr_obj.nsr_name, cfg_delay)
 580                     #     yield from asyncio.sleep(cfg_delay, loop=self._loop)
 581
 582                     for config_attributes_dict in nsr_obj.nsr_cfg_config_attributes_dict.values():
 583                         # Iterate through each priority level
 584                         for vnf_config_attributes_dict in config_attributes_dict:
 585                             # Iterate through each vnfr at this priority level
 586
 587                             # Make up vnf_unique_name with vnfd name and member index
 588                             #vnfr_name = "{}.{}".format(nsr_obj.nsr_name, vnf_config_attributes_dict['name'])
 589                             vnf_unique_name = get_vnf_unique_name(
 590                                 nsr_obj.nsr_name,
 591                                 vnf_config_attributes_dict['name'],
 592                                 str(vnf_config_attributes_dict['member_vnf_index']),
 593                             )
 594                             self._log.info("NS (%s) : VNF (%s) - Processing configuration attributes",
 595                                            nsr_obj.nsr_name, vnf_unique_name)
 596
 597                             # Find vnfr for this vnf_unique_name
 598                             if vnf_unique_name not in nsr_obj._vnfr_dict:
 599                                 self._log.error("NS (%s) - Can not find VNF to be configured: %s", nsr_obj.nsr_name, vnf_unique_name)
 600                             else:
 601                                 # Save this unique VNF's config input parameters
 602                                 nsr_obj.vnf_config_attributes_dict[vnf_unique_name] = vnf_config_attributes_dict
 603                                 nsr_obj.ConfigVNF(nsr_obj._vnfr_dict[vnf_unique_name])
 604
 605                     # Now add the entire NS to the pending config list.
 606                     self._log.info("Scheduling NSR:{} configuration".format(nsr_obj.nsr_name))
 607                     self._parent.add_to_pending(nsr_obj)
 608                     self._parent.add_nsr_obj(nsr_obj)
 609
 610                 except Exception as e:
 611                     self._log.error("Failed processing input parameters for NS (%s) as %s", nsr_obj.nsr_name, str(e))
 612                     raise
 613             else:
 614                 self._log.error("No configuration input parameters for NSR (%s)", nsr_obj.nsr_name)
 615
 616         except Exception as e:
 617             self._log.error("Failed to configure NS (%s) as (%s)", nsr_obj.nsr_name, str(e))
 618             yield from nsr_obj.update_ns_cm_state(conmanY.RecordState.CFG_PROCESS_FAILED)
 619             raise
 620
 621         return True
 622
 623     @asyncio.coroutine
 624     def terminate_NSR(self, id):
 625         nsr_dict = self._nsr_dict
 626         if id not in nsr_dict:
 627             self._log.error("NSR(%s) does not exist!", id)
 628             return
 629         else:
 630             # Remove this NSR if we have it on pending task list
 631             for task in self.pending_tasks:
 632                 if task['nsrid'] == id:
 633                     self.del_from_pending_tasks(task)
 634
 635             # Remove this object from global list
 636             nsr_obj = nsr_dict.pop(id, None)
 637
 638             # Remove this NS cm-state from global status list
 639             self.cm_state['cm_nsr'].remove(nsr_obj.cm_nsr)
 640
 641             # Also remove any scheduled configuration event
 642             for nsr_obj_p in self._parent.pending_cfg:
 643                 if nsr_obj_p == nsr_obj:
 644                     assert id == nsr_obj_p._nsr_id
 645                     #self._parent.pending_cfg.remove(nsr_obj_p)
 646                     # Mark this as being deleted so we do not try to configure it if we are in cfg_delay (will wake up and continue to process otherwise)
 647                     nsr_obj_p.being_deleted = True
 648                     self._log.info("Removed scheduled configuration for NSR(%s)", nsr_obj.nsr_name)
 649
 650             self._parent.remove_nsr_obj(id)
 651
 652             # Call Config Agent to clean up for each VNF
 653             for agent_vnfr in nsr_obj.agent_nsr.vnfrs:
 654                 yield from self._config_agent_mgr.invoke_config_agent_plugins(
 655                     'notify_terminate_vnfr',
 656                     nsr_obj.agent_nsr,
 657                     agent_vnfr)
 658
 659             # publish delete cm-state (cm-nsr)
 660             yield from nsr_obj.delete_cm_nsr()
 661
 662             #####################TBD###########################
 663             # yield from self._config_agent_mgr.invoke_config_agent_plugins('notify_terminate_ns', self.id)
 664
 665             self._log.info("NSR(%s/%s) is deleted", nsr_obj.nsr_name, id)
 666
 667     @asyncio.coroutine
 668     def process_initial_config(self, nsr_obj, conf, script, vnfr_name=None):
 669         '''Apply the initial-config-primitives specified in NSD or VNFD'''
 670
 671         def get_input_file(parameters):
 672             inp = {}
 673
 674             # Add NSR name to file
 675             inp['nsr_name'] = nsr_obj.nsr_name
 676
 677             # Add VNFR name if available
 678             if vnfr_name:
 679                 inp['vnfr_name'] = vnfr_name
 680
 681             # Add parameters for initial config
 682             inp['parameter'] = {}
 683             for parameter in parameters:
 684                 try:
 685                     inp['parameter'][parameter['name']] = parameter['value']
 686                 except KeyError as e:
 687                     if vnfr_name:
 688                         self._log.info("VNFR {} initial config parameter {} with no value: {}".
 689                                        format(vnfr_name, parameter, e))
 690                     else:
 691                         self._log.info("NSR {} initial config parameter {} with no value: {}".
 692                                        format(nsr_obj.nsr_name, parameter, e))
 693
 694
 695             # Add config agents specific to each VNFR
 696             inp['config-agent'] = {}
 697             for vnfr in nsr_obj.agent_nsr.vnfrs:
 698                 # Get the config agent for the VNFR
 699                 # If vnfr name is specified, add only CA specific to that
 700                 if (vnfr_name is None) or \
 701                    (vnfr_name == vnfr.name):
 702                     agent = self._config_agent_mgr.get_vnfr_config_agent(vnfr.vnfr_msg)
 703                     if agent:
 704                         if agent.agent_type != riftcm_config_plugin.DEFAULT_CAP_TYPE:
 705                             inp['config-agent'][vnfr.member_vnf_index] = agent.agent_data
 706                             inp['config-agent'][vnfr.member_vnf_index] \
 707                                 ['service-name'] = agent.get_service_name(vnfr.id)
 708
 709             # Add vnfrs specific data
 710             inp['vnfr'] = {}
 711             for vnfr in nsr_obj.vnfrs:
 712                 v = {}
 713
 714                 v['name'] = vnfr['name']
 715                 v['mgmt_ip_address'] = vnfr['vnf_cfg']['mgmt_ip_address']
 716                 v['mgmt_port'] = vnfr['vnf_cfg']['port']
 717
 718                 if 'dashboard_url' in vnfr:
 719                     v['dashboard_url'] = vnfr['dashboard_url']
 720
 721                 if 'connection_point' in vnfr:
 722                     v['connection_point'] = []
 723                     for cp in vnfr['connection_point']:
 724                         v['connection_point'].append(
 725                             {
 726                                 'name': cp['name'],
 727                                 'ip_address': cp['ip_address'],
 728                             }
 729                         )
 730
 731                 v['vdur'] = []
 732                 vdu_data = []
 733                 for vdu in vnfr['vdur']:
 734                     d = {}
 735                     for k in ['name','management_ip', 'vm_management_ip', 'id']:
 736                         if k in vdu:
 737                             d[k] = vdu[k]
 738                     vdu_data.append(d)
 739                 v['vdur'].append(vdu_data)
 740
 741                 inp['vnfr'][vnfr['member_vnf_index_ref']] = v
 742
 743
 744             self._log.debug("Input data for {}: {}".
 745                             format((vnfr_name if vnfr_name else nsr_obj.nsr_name),
 746                                    inp))
 747
 748             # Convert to YAML string
 749             yaml_string = yaml.dump(inp, default_flow_style=False)
 750
 751             # Write the inputs as yaml file
 752             tmp_file = None
 753             with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
 754                 tmp_file.write(yaml_string.encode("UTF-8"))
 755             self._log.debug("Input file created for {}: {}".
 756                             format((vnfr_name if vnfr_name \
 757                                     else nsr_obj.nsr_name),
 758                                    tmp_file.name))
 759
 760             return tmp_file.name
 761
 762         parameters = []
 763         try:
 764             parameters = conf['parameter']
 765         except Exception as e:
 766             self._log.debug("Parameter conf: {}, e: {}".
 767                             format(conf, e))
 768
 769         inp_file = get_input_file(parameters)
 770
 771         cmd = "{0} {1}".format(script, inp_file)
 772         self._log.debug("Running the CMD: {}".format(cmd))
 773
 774         process = yield from asyncio.create_subprocess_shell(cmd,
 775                                                              loop=self._loop,
 776                                                              stdout=subprocess.PIPE,
 777                                                              stderr=subprocess.PIPE)
 778         stdout, stderr = yield from process.communicate()
 779         rc = yield from process.wait()
 780
 781         if rc:
 782             msg = "NSR/VNFR {} initial config using {} failed with {}: {}". \
 783                   format(vnfr_name if vnfr_name else nsr_obj.nsr_name,
 784                          script, rc, stderr)
 785             self._log.error(msg)
 786             raise InitialConfigError(msg)
 787
 788         try:
 789             os.remove(inp_file)
 790         except Exception as e:
 791             self._log.debug("Error removing input file {}: {}".
 792                             format(inp_file, e))
 793
 794     def get_script_file(self, script_name, d_name, d_id, d_type):
 795           # Get the full path to the script
 796           script = ''
 797           # If script name starts with /, assume it is full path
 798           if script_name[0] == '/':
 799               # The script has full path, use as is
 800               script = script_name
 801           else:
 802               script = os.path.join(os.environ['RIFT_ARTIFACTS'],
 803                                     'launchpad/packages',
 804                                     d_type,
 805                                     d_id,
 806                                     d_name,
 807                                     'scripts',
 808                                     script_name)
 809               self._log.debug("Checking for script at %s", script)
 810               if not os.path.exists(script):
 811                   self._log.warning("Did not find script %s", script)
 812                   script = os.path.join(os.environ['RIFT_INSTALL'],
 813                                         'usr/bin',
 814                                         script_name)
 815
 816               # Seen cases in jenkins, where the script execution fails
 817               # with permission denied. Setting the permission on script
 818               # to make sure it has execute permission
 819               perm = os.stat(script).st_mode
 820               if not (perm  &  stat.S_IXUSR):
 821                   self._log.warning("NSR/VNFR {} initial config script {} " \
 822                                     "without execute permission: {}".
 823                                     format(d_name, script, perm))
 824                   os.chmod(script, perm | stat.S_IXUSR)
 825               return script
 826
 827     @asyncio.coroutine
 828     def process_ns_initial_config(self, nsr_obj):
 829         '''Apply the initial-config-primitives specified in NSD'''
 830
 831         nsr = yield from self.cmdts_obj.get_nsr(nsr_obj.nsr_id)
 832         if 'initial_config_primitive' not in nsr:
 833             return
 834
 835         if nsr is not None:
 836             nsd = yield from self.cmdts_obj.get_nsd(nsr_obj.nsr_id)
 837             for conf in nsr['initial_config_primitive']:
 838                 self._log.debug("NSR {} initial config: {}".
 839                                 format(nsr_obj.nsr_name, conf))
 840                 script = self.get_script_file(conf['user_defined_script'],
 841                                               nsd.name,
 842                                               nsd.id,
 843                                               'nsd')
 844
 845                 yield from self.process_initial_config(nsr_obj, conf, script)
 846
 847     @asyncio.coroutine
 848     def process_vnf_initial_config(self, nsr_obj, vnfr):
 849         '''Apply the initial-config-primitives specified in VNFD'''
 850
 851         vnfr_name = vnfr.name
 852
 853         vnfd = vnfr.vnfd
 854         vnf_cfg = vnfd.vnf_configuration
 855
 856         for conf in vnf_cfg.initial_config_primitive:
 857                 self._log.debug("VNFR {} initial config: {}".
 858                                 format(vnfr_name, conf))
 859
 860                 if not conf.user_defined_script:
 861                     self._log.debug("VNFR {} did not fine user defined script: {}".
 862                                     format(vnfr_name, conf))
 863                     continue
 864
 865                 script = self.get_script_file(conf.user_defined_script,
 866                                               vnfd.name,
 867                                               vnfd.id,
 868                                               'vnfd')
 869
 870                 yield from self.process_initial_config(nsr_obj,
 871                                                        conf.as_dict(),
 872                                                        script,
 873                                                        vnfr_name=vnfr_name)
 874
 875
 876 class ConfigManagerNSR(object):
 877     def __init__(self, log, loop, parent, id):
 878         self._log = log
 879         self._loop = loop
 880         self._rwcal = None
 881         self._vnfr_dict = {}
 882         self._cp_dict = {}
 883         self._nsr_id = id
 884         self._parent = parent
 885         self._log.info("Instantiated NSR entry for id=%s", id)
 886         self.nsr_cfg_config_attributes_dict = {}
 887         self.vnf_config_attributes_dict = {}
 888         self.num_vnfs_to_cfg = 0
 889         self._vnfr_list = []
 890         self.vnf_cfg_list = []
 891         self.this_nsr_dir = None
 892         self.being_deleted = False
 893         self.dts_obj = self._parent.cmdts_obj
 894
 895         # Initialize cm-state for this NS
 896         self.cm_nsr = {}
 897         self.cm_nsr['cm_vnfr'] = []
 898         self.cm_nsr['id'] = id
 899         self.cm_nsr['state'] = self.state_to_string(conmanY.RecordState.INIT)
 900         self.cm_nsr['state_details'] = None
 901
 902         self.set_nsr_name('Not Set')
 903
 904         # Add this NSR cm-state object to global cm-state
 905         parent.cm_state['cm_nsr'].append(self.cm_nsr)
 906
 907         # Place holders for NSR & VNFR classes
 908         self.agent_nsr = None
 909
 910     @property
 911     def nsr_opdata_xpath(self):
 912         ''' Returns full xpath for this NSR cm-state opdata '''
 913         return(
 914             "D,/rw-conman:cm-state" +
 915             "/rw-conman:cm-nsr[rw-conman:id='{}']"
 916         ).format(self._nsr_id)
 917
 918     @property
 919     def vnfrs(self):
 920         return self._vnfr_list
 921
 922     @property
 923     def parent(self):
 924         return self._parent
 925
 926     @property
 927     def nsr_id(self):
 928         return self._nsr_id
 929
 930     @asyncio.coroutine
 931     def publish_cm_state(self):
 932         ''' This function publishes cm_state for this NSR '''
 933
 934         cm_state = conmanY.CmOpdata()
 935         cm_state_nsr = cm_state.cm_nsr.add()
 936         cm_state_nsr.from_dict(self.cm_nsr)
 937         #with self._dts.transaction() as xact:
 938         yield from self.dts_obj.update(self.nsr_opdata_xpath, cm_state_nsr)
 939         self._log.info("Published cm-state with xpath %s and nsr %s",
 940                        self.nsr_opdata_xpath,
 941                        cm_state_nsr)
 942
 943     @asyncio.coroutine
 944     def delete_cm_nsr(self):
 945         ''' This function publishes cm_state for this NSR '''
 946
 947         yield from self.dts_obj.delete(self.nsr_opdata_xpath)
 948         self._log.info("Deleted cm-nsr with xpath %s",
 949                        self.nsr_opdata_xpath)
 950
 951     def set_nsr_name(self, name):
 952         self.nsr_name = name
 953         self.cm_nsr['name'] = name
 954
 955     def set_config_dir(self, caller):
 956         self.this_nsr_dir = os.path.join(
 957             caller._parent.cfg_dir, self.nsr_name, caller._nsr['name_ref'])
 958         if not os.path.exists(self.this_nsr_dir):
 959             os.makedirs(self.this_nsr_dir)
 960             self._log.debug("NSR:(%s), Created configuration directory(%s)",
 961                             caller._nsr['name_ref'], self.this_nsr_dir)
 962         self.config_attributes_file = os.path.join(self.this_nsr_dir, "configuration_config_attributes.yml")
 963         self.xlate_dict_file = os.path.join(self.this_nsr_dir, "nsr_xlate_dict.yml")
 964
 965     def xlate_conf(self, vnfr, vnf_cfg):
 966
 967         # If configuration type is not already set, try to read from attributes
 968         if vnf_cfg['interface_type'] is None:
 969             # Prepare unique name for this VNF
 970             vnf_unique_name = get_vnf_unique_name(
 971                     vnf_cfg['nsr_name'],
 972                     vnfr['short_name'],
 973                     vnfr['member_vnf_index_ref'],
 974                     )
 975
 976             # Find this particular (unique) VNF's config attributes
 977             if (vnf_unique_name in self.vnf_config_attributes_dict):
 978                 vnf_cfg_config_attributes_dict = self.vnf_config_attributes_dict[vnf_unique_name]
 979                 vnf_cfg['interface_type'] = vnf_cfg_config_attributes_dict['configuration_type']
 980                 if 'configuration_options' in vnf_cfg_config_attributes_dict:
 981                     cfg_opts = vnf_cfg_config_attributes_dict['configuration_options']
 982                     for key, value in cfg_opts.items():
 983                         vnf_cfg[key] = value
 984
 985         cfg_path_prefix = '{}/{}/{}_{}'.format(
 986                 self._parent._parent.cfg_dir,
 987                 vnf_cfg['nsr_name'],
 988                 vnfr['short_name'],
 989                 vnfr['member_vnf_index_ref'],
 990                 )
 991
 992         vnf_cfg['cfg_template'] = '{}_{}_template.cfg'.format(cfg_path_prefix, vnf_cfg['interface_type'])
 993         vnf_cfg['cfg_file'] = '{}.cfg'.format(cfg_path_prefix)
 994         vnf_cfg['xlate_script'] = self._parent._parent.cfg_dir + '/xlate_cfg.py'
 995
 996         self._log.debug("VNF endpoint so far: %s", vnf_cfg)
 997
 998         self._log.info("Checking cfg_template %s", vnf_cfg['cfg_template'])
 999         if os.path.exists(vnf_cfg['cfg_template']):
1000             return True
1001         return False
1002
1003     def ConfigVNF(self, vnfr):
1004
1005         vnf_cfg = vnfr['vnf_cfg']
1006         vnf_cm_state = self.find_or_create_vnfr_cm_state(vnf_cfg)
1007
1008         if (vnf_cm_state['state'] == self.state_to_string(conmanY.RecordState.READY_NO_CFG)
1009             or
1010             vnf_cm_state['state'] == self.state_to_string(conmanY.RecordState.READY)):
1011             self._log.warning("NS/VNF (%s/%s) is already configured! Skipped.", self.nsr_name, vnfr['short_name'])
1012             return
1013
1014         #UPdate VNF state
1015         vnf_cm_state['state'] = self.state_to_string(conmanY.RecordState.CFG_PROCESS)
1016
1017         # Now translate the configuration for iP addresses
1018         try:
1019             # Add cp_dict members (TAGS) for this VNF
1020             self._cp_dict['rw_mgmt_ip'] = vnf_cfg['mgmt_ip_address']
1021             self._cp_dict['rw_username'] = vnf_cfg['username']
1022             self._cp_dict['rw_password'] = vnf_cfg['password']
1023             ############################################################
1024             # TBD - Need to lookup above 3 for a given VNF, not global #
1025             # Once we do that no need to dump below file again before  #
1026             # each VNF configuration translation.                      #
1027             # This will require all existing config templates to be    #
1028             # changed for above three tags to include member index     #
1029             ############################################################
1030             try:
1031                 nsr_obj = vnf_cfg['nsr_obj']
1032                 # Generate config_config_attributes.yaml (For debug reference)
1033                 with open(nsr_obj.xlate_dict_file, "w") as yf:
1034                     yf.write(yaml.dump(nsr_obj._cp_dict, default_flow_style=False))
1035             except Exception as e:
1036                 self._log.error("NS:(%s) failed to write nsr xlate tags file as (%s)", nsr_obj.nsr_name, str(e))
1037
1038             if 'cfg_template' in vnf_cfg:
1039                 script_cmd = 'python3 {} -i {} -o {} -x "{}"'.format(vnf_cfg['xlate_script'], vnf_cfg['cfg_template'], vnf_cfg['cfg_file'], self.xlate_dict_file)
1040                 self._log.debug("xlate script command (%s)", script_cmd)
1041                 #xlate_msg = subprocess.check_output(script_cmd).decode('utf-8')
1042                 xlate_msg = subprocess.check_output(script_cmd, shell=True).decode('utf-8')
1043                 self._log.info("xlate script output (%s)", xlate_msg)
1044         except Exception as e:
1045             vnf_cm_state['state'] = self.state_to_string(conmanY.RecordState.CFG_PROCESS_FAILED)
1046             self._log.error("Failed to execute translation script for VNF: %s with (%s)", log_this_vnf(vnf_cfg), str(e))
1047             return
1048
1049         self._log.info("Applying config to VNF: %s = %s!", log_this_vnf(vnf_cfg), vnf_cfg)
1050         try:
1051             #self.vnf_cfg_list.append(vnf_cfg)
1052             self._log.debug("Scheduled configuration!")
1053             vnf_cm_state['state'] = self.state_to_string(conmanY.RecordState.CFG_SCHED)
1054         except Exception as e:
1055             self._log.error("Failed apply_vnf_config to VNF: %s as (%s)", log_this_vnf(vnf_cfg), str(e))
1056             vnf_cm_state['state'] = self.state_to_string(conmanY.RecordState.CFG_PROCESS_FAILED)
1057             raise
1058
1059     def add(self, nsr):
1060         self._log.info("Adding NS Record for id=%s", id)
1061         self._nsr = nsr
1062
1063     def sample_cm_state(self):
1064         return (
1065             {
1066                 'cm_nsr': [
1067                     {
1068                         'cm_vnfr': [
1069                             {
1070                                 'cfg_location': 'location1',
1071                                 'cfg_type': 'script',
1072                                 'connection_point': [
1073                                     {'ip_address': '1.1.1.1', 'name': 'vnf1cp1'},
1074                                     {'ip_address': '1.1.1.2', 'name': 'vnf1cp2'}
1075                                 ],
1076                                 'id': 'vnfrid1',
1077                                 'mgmt_interface': {'ip_address': '7.1.1.1',
1078                                                    'port': 1001},
1079                                 'name': 'vnfrname1',
1080                                 'state': 'init'
1081                             },
1082                             {
1083                                 'cfg_location': 'location2',
1084                                 'cfg_type': 'netconf',
1085                                 'connection_point': [{'ip_address': '2.1.1.1', 'name': 'vnf2cp1'},
1086                                                      {'ip_address': '2.1.1.2', 'name': 'vnf2cp2'}],
1087                                 'id': 'vnfrid2',
1088                                 'mgmt_interface': {'ip_address': '7.1.1.2',
1089                                                    'port': 1001},
1090                                 'name': 'vnfrname2',
1091                                 'state': 'init'}
1092                         ],
1093                         'id': 'nsrid1',
1094                         'name': 'nsrname1',
1095                         'state': 'init'}
1096                 ],
1097                 'states': 'Initialized, '
1098             })
1099
1100     def populate_vm_state_from_vnf_cfg(self):
1101         # Fill in each VNFR from this nsr object
1102         vnfr_list = self._vnfr_list
1103         for vnfr in vnfr_list:
1104             vnf_cfg = vnfr['vnf_cfg']
1105             vnf_cm_state = self.find_vnfr_cm_state(vnfr['id'])
1106
1107             if vnf_cm_state:
1108                 # Fill in VNF management interface
1109                 vnf_cm_state['mgmt_interface']['ip_address'] = vnf_cfg['mgmt_ip_address']
1110                 vnf_cm_state['mgmt_interface']['port'] = vnf_cfg['port']
1111
1112                 # Fill in VNF configuration details
1113                 vnf_cm_state['cfg_type'] = vnf_cfg['config_method']
1114                 vnf_cm_state['cfg_location'] = vnf_cfg['cfg_file']
1115
1116                 # Fill in each connection-point for this VNF
1117                 if "connection_point" in vnfr:
1118                     cp_list = vnfr['connection_point']
1119                     for cp_item_dict in cp_list:
1120                         vnf_cm_state['connection_point'].append(
1121                             {
1122                                 'name' : cp_item_dict['name'],
1123                                 'ip_address' : cp_item_dict['ip_address'],
1124                             }
1125                         )
1126
1127     def state_to_string(self, state):
1128         state_dict = {
1129             conmanY.RecordState.INIT : "init",
1130             conmanY.RecordState.RECEIVED : "received",
1131             conmanY.RecordState.CFG_PROCESS : "cfg_process",
1132             conmanY.RecordState.CFG_PROCESS_FAILED : "cfg_process_failed",
1133             conmanY.RecordState.CFG_SCHED : "cfg_sched",
1134             conmanY.RecordState.CFG_DELAY : "cfg_delay",
1135             conmanY.RecordState.CONNECTING : "connecting",
1136             conmanY.RecordState.FAILED_CONNECTION : "failed_connection",
1137             conmanY.RecordState.NETCONF_CONNECTED : "netconf_connected",
1138             conmanY.RecordState.NETCONF_SSH_CONNECTED : "netconf_ssh_connected",
1139             conmanY.RecordState.RESTCONF_CONNECTED : "restconf_connected",
1140             conmanY.RecordState.CFG_SEND : "cfg_send",
1141             conmanY.RecordState.CFG_FAILED : "cfg_failed",
1142             conmanY.RecordState.READY_NO_CFG : "ready_no_cfg",
1143             conmanY.RecordState.READY : "ready",
1144         }
1145         return state_dict[state]
1146
1147     def find_vnfr_cm_state(self, id):
1148         if self.cm_nsr['cm_vnfr']:
1149             for vnf_cm_state in self.cm_nsr['cm_vnfr']:
1150                 if vnf_cm_state['id'] == id:
1151                     return vnf_cm_state
1152         return None
1153
1154     def find_or_create_vnfr_cm_state(self, vnf_cfg):
1155         vnfr = vnf_cfg['vnfr']
1156         vnf_cm_state = self.find_vnfr_cm_state(vnfr['id'])
1157
1158         if vnf_cm_state is None:
1159             # Not found, Create and Initialize this VNF cm-state
1160             vnf_cm_state = {
1161                 'id' : vnfr['id'],
1162                 'name' : vnfr['short_name'],
1163                 'state' : self.state_to_string(conmanY.RecordState.RECEIVED),
1164                 'mgmt_interface' :
1165                 {
1166                     'ip_address' : vnf_cfg['mgmt_ip_address'],
1167                     'port' : vnf_cfg['port'],
1168                 },
1169                 'cfg_type' : vnf_cfg['config_method'],
1170                 'cfg_location' : vnf_cfg['cfg_file'],
1171                 'connection_point' : [],
1172             }
1173             self.cm_nsr['cm_vnfr'].append(vnf_cm_state)
1174
1175             # Publish newly created cm-state
1176
1177
1178         return vnf_cm_state
1179
1180     @asyncio.coroutine
1181     def get_vnf_cm_state(self, vnfr):
1182         if vnfr:
1183             vnf_cm_state = self.find_vnfr_cm_state(vnfr['id'])
1184             if vnf_cm_state:
1185                 return vnf_cm_state['state']
1186         return False
1187
1188     @asyncio.coroutine
1189     def update_vnf_cm_state(self, vnfr, state):
1190         if vnfr:
1191             vnf_cm_state = self.find_vnfr_cm_state(vnfr['id'])
1192             if vnf_cm_state is None:
1193                 self._log.error("No opdata found for NS/VNF:%s/%s!",
1194                                 self.nsr_name, vnfr['short_name'])
1195                 return
1196
1197             if vnf_cm_state['state'] != self.state_to_string(state):
1198                 old_state = vnf_cm_state['state']
1199                 vnf_cm_state['state'] = self.state_to_string(state)
1200                 # Publish new state
1201                 yield from self.publish_cm_state()
1202                 self._log.info("VNF ({}/{}/{}) state change: {} -> {}"
1203                                .format(self.nsr_name,
1204                                        vnfr['short_name'],
1205                                        vnfr['member_vnf_index_ref'],
1206                                        old_state,
1207                                        vnf_cm_state['state']))
1208
1209         else:
1210             self._log.error("No VNFR supplied for state update (NS=%s)!",
1211                             self.nsr_name)
1212
1213     @property
1214     def get_ns_cm_state(self):
1215         return self.cm_nsr['state']
1216
1217     @asyncio.coroutine
1218     def update_ns_cm_state(self, state, state_details=None):
1219         if self.cm_nsr['state'] != self.state_to_string(state):
1220             old_state = self.cm_nsr['state']
1221             self.cm_nsr['state'] = self.state_to_string(state)
1222             self.cm_nsr['state_details'] = state_details if state_details is not None else None
1223             self._log.info("NS ({}) state change: {} -> {}"
1224                            .format(self.nsr_name,
1225                                    old_state,
1226                                    self.cm_nsr['state']))
1227             # Publish new state
1228             yield from self.publish_cm_state()
1229
1230     @asyncio.coroutine
1231     def add_vnfr(self, vnfr, vnfr_msg):
1232
1233         @asyncio.coroutine
1234         def populate_subnets_from_vlr(id):
1235             try:
1236                 # Populate cp_dict with VLR subnet info
1237                 vlr = yield from self.dts_obj.get_vlr(id)
1238                 if vlr is not None and 'assigned_subnet' in vlr:
1239                     subnet = {vlr.name:vlr.assigned_subnet}
1240                     self._cp_dict[vnfr['member_vnf_index_ref']].update(subnet)
1241                     self._cp_dict.update(subnet)
1242                     self._log.debug("VNF:(%s) Updated assigned subnet = %s",
1243                                     vnfr['short_name'], subnet)
1244             except Exception as e:
1245                 self._log.error("VNF:(%s) VLR Error = %s",
1246                                 vnfr['short_name'], e)
1247
1248         if vnfr['id'] not in self._vnfr_dict:
1249             self._log.info("NSR(%s) : Adding VNF Record for name=%s, id=%s", self._nsr_id, vnfr['short_name'], vnfr['id'])
1250             # Add this vnfr to the list for show, or single traversal
1251             self._vnfr_list.append(vnfr)
1252         else:
1253             self._log.warning("NSR(%s) : VNF Record for name=%s, id=%s already exists, overwriting", self._nsr_id, vnfr['short_name'], vnfr['id'])
1254
1255         # Make vnfr available by id as well as by name
1256         unique_name = get_vnf_unique_name(self.nsr_name, vnfr['short_name'], vnfr['member_vnf_index_ref'])
1257         self._vnfr_dict[unique_name] = vnfr
1258         self._vnfr_dict[vnfr['id']] = vnfr
1259
1260         # Create vnf_cfg dictionary with default values
1261         vnf_cfg = {
1262             'nsr_obj' : self,
1263             'vnfr' : vnfr,
1264             'agent_vnfr' : self.agent_nsr.add_vnfr(vnfr, vnfr_msg),
1265             'nsr_name' : self.nsr_name,
1266             'nsr_id' : self._nsr_id,
1267             'vnfr_name' : vnfr['short_name'],
1268             'member_vnf_index' : vnfr['member_vnf_index_ref'],
1269             'port' : 0,
1270             'username' : 'admin',
1271             'password' : 'admin',
1272             'config_method' : 'None',
1273             'protocol' : 'None',
1274             'mgmt_ip_address' : '0.0.0.0',
1275             'cfg_file' : 'None',
1276             'cfg_retries' : 0,
1277             'script_type' : 'bash',
1278         }
1279
1280         # Update the mgmt ip address
1281         # In case the config method is none, this is not
1282         # updated later
1283         try:
1284             vnf_cfg['mgmt_ip_address'] = vnfr_msg.mgmt_interface.ip_address
1285             vnf_cfg['port'] = vnfr_msg.mgmt_interface.port
1286         except Exception as e:
1287             self._log.warn(
1288                 "VNFR {}({}), unable to retrieve mgmt ip address: {}".
1289                 format(vnfr['short_name'], vnfr['id'], e))
1290
1291         vnfr['vnf_cfg'] = vnf_cfg
1292         self.find_or_create_vnfr_cm_state(vnf_cfg)
1293
1294         '''
1295         Build the connection-points list for this VNF (self._cp_dict)
1296         '''
1297         # Populate global CP list self._cp_dict from VNFR
1298         cp_list = []
1299         if 'connection_point' in vnfr:
1300             cp_list = vnfr['connection_point']
1301
1302         self._cp_dict[vnfr['member_vnf_index_ref']] = {}
1303         if 'vdur' in vnfr:
1304             for vdur in vnfr['vdur']:
1305                 if 'internal_connection_point' in vdur:
1306                     cp_list += vdur['internal_connection_point']
1307
1308                 for cp_item_dict in cp_list:
1309                     # Populate global dictionary
1310                     self._cp_dict[
1311                         cp_item_dict['name']
1312                     ] = cp_item_dict['ip_address']
1313
1314                     # Populate unique member specific dictionary
1315                     self._cp_dict[
1316                         vnfr['member_vnf_index_ref']
1317                     ][
1318                         cp_item_dict['name']
1319                     ] = cp_item_dict['ip_address']
1320
1321                     # Fill in the subnets from vlr
1322                     if 'vlr_ref' in cp_item_dict:
1323                         ### HACK: Internal connection_point do not have VLR reference
1324                         yield from populate_subnets_from_vlr(cp_item_dict['vlr_ref'])
1325
1326         if 'internal_vlr' in vnfr:
1327             for ivlr in vnfr['internal_vlr']:
1328                 yield from populate_subnets_from_vlr(ivlr['vlr_ref'])
1329
1330         # Update vnfr
1331         vnf_cfg['agent_vnfr']._vnfr = vnfr
1332         return vnf_cfg['agent_vnfr']
1333
1334
1335 class XPaths(object):
1336     @staticmethod
1337     def nsr_opdata(k=None):
1338         return ("D,/nsr:ns-instance-opdata/nsr:nsr" +
1339                 ("[nsr:ns-instance-config-ref='{}']".format(k) if k is not None else ""))
1340
1341     @staticmethod
1342     def nsd_msg(k=None):
1343         return ("C,/nsd:nsd-catalog/nsd:nsd" +
1344                 "[nsd:id = '{}']".format(k) if k is not None else "")
1345
1346     @staticmethod
1347     def vnfr_opdata(k=None):
1348         return ("D,/vnfr:vnfr-catalog/vnfr:vnfr" +
1349                 ("[vnfr:id='{}']".format(k) if k is not None else ""))
1350
1351     @staticmethod
1352     def vnfd(k=None):
1353         return ("C,/vnfd:vnfd-catalog/vnfd:vnfd" +
1354                 ("[vnfd:id='{}']".format(k) if k is not None else ""))
1355
1356     @staticmethod
1357     def config_agent(k=None):
1358         return ("D,/rw-config-agent:config-agent/rw-config-agent:account" +
1359                 ("[rw-config-agent:name='{}']".format(k) if k is not None else ""))
1360
1361     @staticmethod
1362     def nsr_config(k=None):
1363         return ("C,/nsr:ns-instance-config/nsr:nsr[nsr:id='{}']".format(k) if k is not None else "")
1364
1365     @staticmethod
1366     def vlr(k=None):
1367         return ("D,/vlr:vlr-catalog/vlr:vlr[vlr:id='{}']".format(k) if k is not None else "")
1368
1369 class ConfigManagerDTS(object):
1370     ''' This class either reads from DTS or publishes to DTS '''
1371
1372     def __init__(self, log, loop, parent, dts):
1373         self._log = log
1374         self._loop = loop
1375         self._parent = parent
1376         self._dts = dts
1377
1378     @asyncio.coroutine
1379     def _read_dts(self, xpath, do_trace=False):
1380         self._log.debug("_read_dts path = %s", xpath)
1381         flags = rwdts.XactFlag.MERGE
1382         res_iter = yield from self._dts.query_read(
1383                 xpath, flags=flags
1384                 )
1385
1386         results = []
1387         try:
1388             for i in res_iter:
1389                 result = yield from i
1390                 if result is not None:
1391                     results.append(result.result)
1392         except:
1393             pass
1394
1395         return results
1396
1397
1398     @asyncio.coroutine
1399     def get_nsr(self, id):
1400         self._log.debug("Attempting to get NSR: %s", id)
1401         nsrl = yield from self._read_dts(XPaths.nsr_opdata(id), False)
1402         nsr = None
1403         if len(nsrl) > 0:
1404             nsr =  nsrl[0].as_dict()
1405         return nsr
1406
1407     @asyncio.coroutine
1408     def get_nsr_config(self, id):
1409         self._log.debug("Attempting to get config NSR: %s", id)
1410         nsrl = yield from self._read_dts(XPaths.nsr_config(id), False)
1411         nsr = None
1412         if len(nsrl) > 0:
1413             nsr =  nsrl[0]
1414         return nsr
1415
1416     @asyncio.coroutine
1417     def get_nsd_msg(self, id):
1418         self._log.debug("Attempting to get NSD: %s", id)
1419         nsdl = yield from self._read_dts(XPaths.nsd_msg(id), False)
1420         nsd_msg = None
1421         if len(nsdl) > 0:
1422             nsd_msg =  nsdl[0]
1423         return nsd_msg
1424
1425     @asyncio.coroutine
1426     def get_nsd(self, nsr_id):
1427         self._log.debug("Attempting to get NSD for NSR: %s", id)
1428         nsr_config = yield from self.get_nsr_config(nsr_id)
1429         nsd_msg = nsr_config.nsd
1430         return nsd_msg
1431
1432     @asyncio.coroutine
1433     def get_vnfr(self, id):
1434         self._log.debug("Attempting to get VNFR: %s", id)
1435         vnfrl = yield from self._read_dts(XPaths.vnfr_opdata(id), do_trace=False)
1436         vnfr_msg = None
1437         if len(vnfrl) > 0:
1438             vnfr_msg = vnfrl[0]
1439         return vnfr_msg
1440
1441     @asyncio.coroutine
1442     def get_vnfd(self, vnfd_id):
1443         self._log.debug("Attempting to get VNFD: %s", vnfd_id)
1444         vnfdl = yield from self._read_dts(XPaths.vnfd(vnfd_id), do_trace=False)
1445         vnfd_msg = None
1446         if len(vnfdl) > 0:
1447             vnfd_msg = vnfdl[0]
1448         return vnfd_msg
1449
1450     @asyncio.coroutine
1451     def get_vlr(self, id):
1452         self._log.debug("Attempting to get VLR subnet: %s", id)
1453         vlrl = yield from self._read_dts(XPaths.vlr(id), do_trace=True)
1454         vlr_msg = None
1455         if len(vlrl) > 0:
1456             vlr_msg = vlrl[0]
1457         return vlr_msg
1458
1459     @asyncio.coroutine
1460     def get_config_agents(self, name):
1461         self._log.debug("Attempting to get config_agents: %s", name)
1462         cfgagentl = yield from self._read_dts(XPaths.config_agent(name), False)
1463         return cfgagentl
1464
1465     @asyncio.coroutine
1466     def update(self, path, msg, flags=rwdts.XactFlag.REPLACE):
1467         """
1468         Update a cm-state (cm-nsr) record in DTS with the path and message
1469         """
1470         self._log.debug("Updating cm-state %s:%s dts_pub_hdl = %s", path, msg, self.dts_pub_hdl)
1471         self.dts_pub_hdl.update_element(path, msg, flags)
1472         self._log.debug("Updated cm-state, %s:%s", path, msg)
1473
1474     @asyncio.coroutine
1475     def delete(self, path):
1476         """
1477         Delete cm-nsr record in DTS with the path only
1478         """
1479         self._log.debug("Deleting cm-nsr %s dts_pub_hdl = %s", path, self.dts_pub_hdl)
1480         self.dts_pub_hdl.delete_element(path)
1481         self._log.debug("Deleted cm-nsr, %s", path)
1482
1483     @asyncio.coroutine
1484     def register(self):
1485         yield from self.register_to_publish()
1486         yield from self.register_for_nsr()
1487
1488     @asyncio.coroutine
1489     def register_to_publish(self):
1490         ''' Register to DTS for publishing cm-state opdata '''
1491
1492         xpath = "D,/rw-conman:cm-state/rw-conman:cm-nsr"
1493         self._log.debug("Registering to publish cm-state @ %s", xpath)
1494         hdl = rift.tasklets.DTS.RegistrationHandler()
1495         with self._dts.group_create() as group:
1496             self.dts_pub_hdl = group.register(xpath=xpath,
1497                                               handler=hdl,
1498                                               flags=rwdts.Flag.PUBLISHER | rwdts.Flag.NO_PREP_READ)
1499
1500     @property
1501     def nsr_xpath(self):
1502         return "D,/nsr:ns-instance-opdata/nsr:nsr"
1503
1504     @asyncio.coroutine
1505     def register_for_nsr(self):
1506         """ Register for NSR changes """
1507
1508         @asyncio.coroutine
1509         def on_prepare(xact_info, query_action, ks_path, msg):
1510             """ This NSR is created """
1511             self._log.debug("Received NSR instantiate on_prepare (%s:%s:%s)",
1512                             query_action,
1513                             ks_path,
1514                             msg)
1515
1516             if (query_action == rwdts.QueryAction.UPDATE or
1517                 query_action == rwdts.QueryAction.CREATE):
1518                 msg_dict = msg.as_dict()
1519                 # Update Each NSR/VNFR state)
1520                 if ('operational_status' in msg_dict and
1521                     msg_dict['operational_status'] == 'running'):
1522                     # Add to the task list
1523                     self._parent.add_to_pending_tasks({'nsrid' : msg_dict['ns_instance_config_ref'], 'retries' : 5})
1524             elif query_action == rwdts.QueryAction.DELETE:
1525                 nsr_id = msg.ns_instance_config_ref
1526                 asyncio.ensure_future(self._parent.terminate_NSR(nsr_id), loop=self._loop)
1527             else:
1528                 raise NotImplementedError(
1529                     "%s action on cm-state not supported",
1530                     query_action)
1531
1532             xact_info.respond_xpath(rwdts.XactRspCode.ACK)
1533
1534         try:
1535             handler = rift.tasklets.DTS.RegistrationHandler(on_prepare=on_prepare)
1536             self.dts_reg_hdl = yield from self._dts.register(self.nsr_xpath,
1537                                                              flags=rwdts.Flag.SUBSCRIBER | rwdts.Flag.DELTA_READY,
1538                                                              handler=handler)
1539         except Exception as e:
1540             self._log.error("Failed to register for NSR changes as %s", str(e))
1541
1542