osm_pla/server/server.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Copyright 2020 ArctosLabs Scandinavia AB
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  15 # implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18
  19 import asyncio
  20 import logging
  21 import itertools
  22 from pathlib import Path
  23
  24 import yaml
  25 from osm_common import dbmemory, dbmongo, msglocal, msgkafka
  26
  27 from osm_pla.config.config import Config
  28 from osm_pla.placement.mznplacement import MznPlacementConductor
  29 from osm_pla.placement.mznplacement import NsPlacementDataFactory
  30
  31
  32 class Server:
  33     pil_price_list_file = Path('/placement/pil_price_list.yaml')
  34     vnf_price_list_file = Path('/placement/vnf_price_list.yaml')
  35
  36     def __init__(self, config: Config, loop=None):
  37         self.log = logging.getLogger("pla.server")
  38         self.db = None
  39         self.msgBus = None
  40         self.config = config
  41         self.loop = loop or asyncio.get_event_loop()
  42
  43         try:
  44             if config.get('database', 'driver') == "mongo":
  45                 self.db = dbmongo.DbMongo()
  46                 self.db.db_connect(config.get('database'))
  47             elif config.get('database', 'driver') == "memory":
  48                 self.db = dbmemory.DbMemory()
  49                 self.db.db_connect(config.get('database'))
  50             else:
  51                 raise Exception("Invalid configuration param '{}' at '[database]':'driver'".format(
  52                     config.get('database', 'driver')))
  53
  54             if config.get('message', 'driver') == "local":
  55                 self.msgBus = msglocal.MsgLocal()
  56             elif config.get('message', 'driver') == "kafka":
  57                 self.msgBus = msgkafka.MsgKafka()
  58             else:
  59                 raise Exception("Invalid message bus driver {}".format(
  60                     config.get('message', 'driver')))
  61             self.msgBus.loop = loop
  62             self.msgBus.connect(config.get('message'))
  63
  64         except Exception as e:
  65             self.log.exception("kafka setup error. Exception: {}".format(e))
  66
  67     def _get_nslcmop(self, nsdlcmop_id):
  68         """
  69         :param nsdlcmop_id:
  70         :return: nslcmop from database corresponding to nslcmop_id
  71         """
  72         db_filter = {"_id": nsdlcmop_id}
  73         nslcmop = self.db.get_one("nslcmops", db_filter)
  74         return nslcmop
  75
  76     def _get_projects(self):
  77         """
  78         :return: project name to project id mapping
  79         """
  80         projects = self.db.get_list("projects")
  81         return {project['_id']: project['name'] for project in projects}
  82
  83     def _get_nsd(self, nsd_id):
  84         """
  85         :param nsd_id:
  86         :return: nsd from database corresponding to nsd_id
  87         """
  88         db_filter = {"_id": nsd_id}
  89         return self.db.get_one("nsds", db_filter)
  90
  91     def _get_vim_accounts(self, vim_account_ids):
  92         """
  93         :param vim_account_ids: list of VIM account ids
  94         :return: list of vim account entries from database corresponding to list in vim_accounts_id
  95         """
  96         db_filter = {"_id": vim_account_ids}
  97         return self.db.get_list("vim_accounts", db_filter)
  98
  99     def _read_vnf_price_list(self, price_list_file_path):
 100         """
 101         read vnf price list configuration file
 102         :param price_list_file_path:
 103         :return:
 104         """
 105         with open(str(price_list_file_path)) as pl_fd:
 106             price_list = yaml.safe_load_all(pl_fd)
 107             return next(price_list)
 108
 109     def _price_list_with_project(self, price_list):
 110         """
 111         Figure out if this price list is with  project or not.
 112         Note: to handle the unlikely event that a project is called 'prices' we do not simply check if 'prices'
 113         is in the dict keys for a price list sequence but rather go down one step in the nesting
 114         in which we either have
 115         1) 'prices:{vim_url:...}' if prices are also per project, or
 116         2) '{vim_url:...}' if prices are only per vim
 117
 118         :param price_list:
 119         :return: True if project part of price list, else False
 120         """
 121         price_list_entry_keys = set(price_list[0].keys())
 122         price_list_entry_keys.remove('vnfd')
 123         pl_key = price_list_entry_keys.pop()
 124         entry_to_check = price_list[0][pl_key][0].keys()
 125         return True if 'prices' in entry_to_check else False
 126
 127     def _get_vnf_price_list(self, price_list_file_path, project_name=None):
 128         """
 129         read vnf price list configuration file, determine its type and reformat content accordingly
 130
 131         :param price_list_file_path:
 132         :param project_name:
 133         :return: dictionary formatted as {'<vnfd>': {'<vim-url>':'<price>'}}
 134         """
 135         price_list_data = self._read_vnf_price_list(price_list_file_path)
 136         if self._price_list_with_project(price_list_data):
 137             res = {}
 138             for i in price_list_data:
 139                 price_data = i[project_name] if type(i[project_name]) is dict else i[project_name][0]
 140                 res_component = {i['vim_name']: i['price'] for i in price_data['prices']}
 141                 res.update({i['vnfd']: res_component})
 142             return res
 143         else:
 144             return {i['vnfd']: {i1['vim_name']: i1['price'] for i1 in i['prices']} for i in price_list_data}
 145
 146     def _get_pil_info(self, pil_info_file_path):
 147         """
 148         read and return pil information from file
 149         :param pil_info_file_path: Path to pil_info file
 150         :return pil configuration file content as Python object
 151         """
 152         with open(str(pil_info_file_path)) as pil_fd:
 153             data = yaml.safe_load_all(pil_fd)
 154             return next(data)
 155
 156     def _create_vnf_id_maps(self, nsd):
 157         """
 158         map identifier for 'member-vnf-index' in nsd to syntax that is safe for mzn
 159
 160          return tuples with mappings {<adjusted id>: <original id>} and {<original id>: <adjusted id>}
 161         """
 162         # TODO: Change for multiple DF support
 163         ns_df = nsd.get('df', [{}])[0]
 164         next_idx = itertools.count()
 165         member_vnf_index2mzn = {e['id']: 'VNF' + str(next(next_idx)) for e in
 166                                 ns_df.get('vnf-profile', [])}
 167
 168         # reverse the name map dictionary, used when the placement result is remapped
 169         mzn_name2member_vnf_index = {v: k for k, v in member_vnf_index2mzn.items()}
 170
 171         return member_vnf_index2mzn, mzn_name2member_vnf_index
 172
 173     async def get_placement(self, nslcmop_id):
 174         """
 175         - Collects and prepares placement information.
 176         - Request placement computation.
 177         - Formats and distribute placement result
 178
 179         Note: exceptions result in empty response message
 180
 181         :param nslcmop_id:
 182         :return:
 183         """
 184         try:
 185             nslcmop = self._get_nslcmop(nslcmop_id)
 186             nsd = self._get_nsd(nslcmop['operationParams']['nsdId'])
 187             member_vnf_index2mzn, mzn2member_vnf_index = self._create_vnf_id_maps(nsd)
 188             # adjust vnf identifiers
 189             # TODO: Change for multiple DF support
 190             ns_df = nsd.get('df', [{}])[0]
 191             for vnf_profile in ns_df.get('vnf-profile', []):
 192                 vnf_profile['id'] = member_vnf_index2mzn[vnf_profile['id']]
 193                 for vlc in vnf_profile.get('virtual-link-connectivity', []):
 194                     for ccpd in vlc.get('constituent-cpd-id', []):
 195                         ccpd['constituent-base-element-id'] = member_vnf_index2mzn[ccpd['constituent-base-element-id']]
 196             self.log.info("adjusted nsd: {}".format(nsd))
 197             projects = self._get_projects()
 198             self.log.info("projects: {}".format(projects))
 199             nslcmop_project = nslcmop['_admin']['projects_read'][0]
 200             self.log.info("nslcmop_project: {}".format(nslcmop_project))
 201             valid_vim_accounts = nslcmop['operationParams']['validVimAccounts']
 202             vim_accounts_data = self._get_vim_accounts(valid_vim_accounts)
 203             vims_information = {_['name']: _['_id'] for _ in vim_accounts_data}
 204             price_list = self._get_vnf_price_list(Server.vnf_price_list_file, projects[nslcmop_project])
 205             pil_info = self._get_pil_info(Server.pil_price_list_file)
 206             pinnings = nslcmop['operationParams'].get('vnf', [])
 207             # remap member-vnf-index values according to id map
 208             for pinning in pinnings:
 209                 pinning['member-vnf-index'] = member_vnf_index2mzn[pinning['member-vnf-index']]
 210             self.log.info("pinnings: {}".format(pinnings))
 211             order_constraints = nslcmop['operationParams'].get('placement-constraints')
 212             self.log.info("order constraints: {}".format(order_constraints))
 213
 214             nspd = NsPlacementDataFactory(vims_information,
 215                                           price_list,
 216                                           nsd,
 217                                           pil_info,
 218                                           pinnings, order_constraints).create_ns_placement_data()
 219
 220             vnf_placement = MznPlacementConductor(self.log).do_placement_computation(nspd)
 221
 222         except Exception as e:
 223             # Note: there is no cure for failure so we have a catch-all clause here
 224             self.log.exception("PLA fault. Exception: {}".format(e))
 225             vnf_placement = []
 226         finally:
 227             # remap names in vnf_placement
 228             for e in vnf_placement:
 229                 e['member-vnf-index'] = mzn2member_vnf_index[e['member-vnf-index']]
 230             await self.msgBus.aiowrite("pla", "placement",
 231                                        {'placement': {'vnf': vnf_placement, 'nslcmopId': nslcmop_id}})
 232
 233     def handle_kafka_command(self, topic, command, params):
 234         self.log.info("Kafka msg arrived: {} {} {}".format(topic, command, params))
 235         if topic == "pla" and command == "get_placement":
 236             nslcmop_id = params.get('nslcmopId')
 237             self.loop.create_task(self.get_placement(nslcmop_id))
 238
 239     async def kafka_read(self):
 240         self.log.info("Task kafka_read start")
 241         while True:
 242             try:
 243                 topics = "pla"
 244                 await self.msgBus.aioread(topics, self.loop, self.handle_kafka_command)
 245             except Exception as e:
 246                 self.log.error("kafka read error. Exception: {}".format(e))
 247                 await asyncio.sleep(5, loop=self.loop)
 248
 249     def run(self):
 250         self.loop.run_until_complete(self.kafka_read())
 251         self.loop.close()
 252         self.loop = None
 253         if self.msgBus:
 254             self.msgBus.disconnect()