osm_pla/server/server.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Copyright 2020 ArctosLabs Scandinavia AB
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  15 # implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18
  19 import asyncio
  20 import logging
  21 import itertools
  22 from pathlib import Path
  23
  24 import yaml
  25 from osm_common import dbmemory, dbmongo, msglocal, msgkafka
  26
  27 from osm_pla.config.config import Config
  28 from osm_pla.placement.mznplacement import MznPlacementConductor
  29 from osm_pla.placement.mznplacement import NsPlacementDataFactory
  30
  31
  32 class Server:
  33     pil_price_list_file = Path('/placement/pil_price_list.yaml')
  34     vnf_price_list_file = Path('/placement/vnf_price_list.yaml')
  35
  36     def __init__(self, config: Config, loop=None):
  37         self.log = logging.getLogger("pla.server")
  38         self.db = None
  39         self.msgBus = None
  40         self.config = config
  41         self.loop = loop or asyncio.get_event_loop()
  42
  43         try:
  44             if config.get('database', 'driver') == "mongo":
  45                 self.db = dbmongo.DbMongo()
  46                 self.db.db_connect(config.get('database'))
  47             elif config.get('database', 'driver') == "memory":
  48                 self.db = dbmemory.DbMemory()
  49                 self.db.db_connect(config.get('database'))
  50             else:
  51                 raise Exception("Invalid configuration param '{}' at '[database]':'driver'".format(
  52                     config.get('database', 'driver')))
  53
  54             if config.get('message', 'driver') == "local":
  55                 self.msgBus = msglocal.MsgLocal()
  56             elif config.get('message', 'driver') == "kafka":
  57                 self.msgBus = msgkafka.MsgKafka()
  58             else:
  59                 raise Exception("Invalid message bus driver {}".format(
  60                     config.get('message', 'driver')))
  61             self.msgBus.loop = loop
  62             self.msgBus.connect(config.get('message'))
  63
  64         except Exception as e:
  65             self.log.exception("kafka setup error. Exception: {}".format(e))
  66
  67     def _get_nslcmop(self, nsdlcmop_id):
  68         """
  69         :param nsdlcmop_id:
  70         :return: nslcmop from database corresponding to nslcmop_id
  71         """
  72         db_filter = {"_id": nsdlcmop_id}
  73         nslcmop = self.db.get_one("nslcmops", db_filter)
  74         return nslcmop
  75
  76     def _get_projects(self):
  77         """
  78         :return: project name to project id mapping
  79         """
  80         projects = self.db.get_list("projects")
  81         return {project['_id']: project['name'] for project in projects}
  82
  83     def _get_nsd(self, nsd_id):
  84         """
  85         :param nsd_id:
  86         :return: nsd from database corresponding to nsd_id
  87         """
  88         db_filter = {"_id": nsd_id}
  89         return self.db.get_one("nsds", db_filter)
  90
  91     def _get_vim_accounts(self, vim_account_ids):
  92         """
  93         :param vim_account_ids: list of VIM account ids
  94         :return: list of vim account entries from database corresponding to list in vim_accounts_id
  95         """
  96         db_filter = {"_id": vim_account_ids}
  97         return self.db.get_list("vim_accounts", db_filter)
  98
  99     def _read_vnf_price_list(self, price_list_file_path):
 100         """
 101         read vnf price list configuration file
 102         :param price_list_file_path:
 103         :return:
 104         """
 105         with open(str(price_list_file_path)) as pl_fd:
 106             price_list = yaml.safe_load_all(pl_fd)
 107             return next(price_list)
 108
 109     def _price_list_with_project(self, price_list):
 110         """
 111         Figure out if this price list is with  project or not.
 112         Note: to handle the unlikely event that a project is called 'prices' we do not simply check if 'prices'
 113         is in the dict keys for a price list sequence but rather go down one step in the nesting
 114         in which we either have
 115         1) 'prices:{vim_url:...}' if prices are also per project, or
 116         2) '{vim_url:...}' if prices are only per vim
 117
 118         :param price_list:
 119         :return: True if project part of price list, else False
 120         """
 121         price_list_entry_keys = set(price_list[0].keys())
 122         price_list_entry_keys.remove('vnfd')
 123         pl_key = price_list_entry_keys.pop()
 124         entry_to_check = price_list[0][pl_key][0].keys()
 125         return True if 'prices' in entry_to_check else False
 126
 127     def _get_vnf_price_list(self, price_list_file_path, project_name=None):
 128         """
 129         read vnf price list configuration file, determine its type and reformat content accordingly
 130
 131         :param price_list_file_path:
 132         :param project_name:
 133         :return: dictionary formatted as {'<vnfd>': {'<vim-url>':'<price>'}}
 134         """
 135         price_list_data = self._read_vnf_price_list(price_list_file_path)
 136         if self._price_list_with_project(price_list_data):
 137             res = {}
 138             for i in price_list_data:
 139                 price_data = i[project_name] if type(i[project_name]) is dict else i[project_name][0]
 140                 res_component = {i['vim_name']: i['price'] for i in price_data['prices']}
 141                 res.update({i['vnfd']: res_component})
 142             return res
 143         else:
 144             return {i['vnfd']: {i1['vim_name']: i1['price'] for i1 in i['prices']} for i in price_list_data}
 145
 146     def _get_pil_info(self, pil_info_file_path):
 147         """
 148         read and return pil information from file
 149         :param pil_info_file_path: Path to pil_info file
 150         :return pil configuration file content as Python object
 151         """
 152         with open(str(pil_info_file_path)) as pil_fd:
 153             data = yaml.safe_load_all(pil_fd)
 154             return next(data)
 155
 156     def _create_vnf_id_maps(self, nsd):
 157         """
 158         map identifier for 'member-vnf-index' in nsd to syntax that is safe for mzn
 159
 160          return tuples with mappings {<adjusted id>: <original id>} and {<original id>: <adjusted id>}
 161         """
 162         next_idx = itertools.count()
 163         member_vnf_index2mzn = {e['member-vnf-index']: 'VNF' + str(next(next_idx)) for e in
 164                                 nsd['constituent-vnfd']}
 165
 166         # reverse the name map dictionary, used when the placement result is remapped
 167         mzn_name2member_vnf_index = {v: k for k, v in member_vnf_index2mzn.items()}
 168
 169         return member_vnf_index2mzn, mzn_name2member_vnf_index
 170
 171     async def get_placement(self, nslcmop_id):
 172         """
 173         - Collects and prepares placement information.
 174         - Request placement computation.
 175         - Formats and distribute placement result
 176
 177         Note: exceptions result in empty response message
 178
 179         :param nslcmop_id:
 180         :return:
 181         """
 182         try:
 183             nslcmop = self._get_nslcmop(nslcmop_id)
 184             nsd = self._get_nsd(nslcmop['operationParams']['nsdId'])
 185             member_vnf_index2mzn, mzn2member_vnf_index = self._create_vnf_id_maps(nsd)
 186             # adjust vnf identifiers
 187             for e in nsd['constituent-vnfd']:
 188                 e['member-vnf-index'] = member_vnf_index2mzn[e['member-vnf-index']]
 189             for vld in nsd['vld']:
 190                 for cp_ref in vld['vnfd-connection-point-ref']:
 191                     cp_ref['member-vnf-index-ref'] = member_vnf_index2mzn[cp_ref['member-vnf-index-ref']]
 192             self.log.info("adjusted nsd: {}".format(nsd))
 193             projects = self._get_projects()
 194             self.log.info("projects: {}".format(projects))
 195             nslcmop_project = nslcmop['_admin']['projects_read'][0]
 196             self.log.info("nslcmop_project: {}".format(nslcmop_project))
 197             valid_vim_accounts = nslcmop['operationParams']['validVimAccounts']
 198             vim_accounts_data = self._get_vim_accounts(valid_vim_accounts)
 199             vims_information = {_['name']: _['_id'] for _ in vim_accounts_data}
 200             price_list = self._get_vnf_price_list(Server.vnf_price_list_file, projects[nslcmop_project])
 201             pil_info = self._get_pil_info(Server.pil_price_list_file)
 202             pinnings = nslcmop['operationParams'].get('vnf')
 203             # remap member-vnf-index values according to id map
 204             for pinning in pinnings:
 205                 pinning['member-vnf-index'] = member_vnf_index2mzn[pinning['member-vnf-index']]
 206             self.log.info("pinnings: {}".format(pinnings))
 207             order_constraints = nslcmop['operationParams'].get('placement-constraints')
 208             self.log.info("order constraints: {}".format(order_constraints))
 209
 210             nspd = NsPlacementDataFactory(vims_information,
 211                                           price_list,
 212                                           nsd,
 213                                           pil_info,
 214                                           pinnings, order_constraints).create_ns_placement_data()
 215
 216             vnf_placement = MznPlacementConductor(self.log).do_placement_computation(nspd)
 217
 218         except Exception as e:
 219             # Note: there is no cure for failure so we have a catch-all clause here
 220             self.log.exception("PLA fault. Exception: {}".format(e))
 221             vnf_placement = []
 222         finally:
 223             # remap names in vnf_placement
 224             for e in vnf_placement:
 225                 e['member-vnf-index'] = mzn2member_vnf_index[e['member-vnf-index']]
 226             await self.msgBus.aiowrite("pla", "placement",
 227                                        {'placement': {'vnf': vnf_placement, 'nslcmopId': nslcmop_id}})
 228
 229     def handle_kafka_command(self, topic, command, params):
 230         self.log.info("Kafka msg arrived: {} {} {}".format(topic, command, params))
 231         if topic == "pla" and command == "get_placement":
 232             nslcmop_id = params.get('nslcmopId')
 233             self.loop.create_task(self.get_placement(nslcmop_id))
 234
 235     async def kafka_read(self):
 236         self.log.info("Task kafka_read start")
 237         while True:
 238             try:
 239                 topics = "pla"
 240                 await self.msgBus.aioread(topics, self.loop, self.handle_kafka_command)
 241             except Exception as e:
 242                 self.log.error("kafka read error. Exception: {}".format(e))
 243                 await asyncio.sleep(5, loop=self.loop)
 244
 245     def run(self):
 246         self.loop.run_until_complete(self.kafka_read())
 247         self.loop.close()
 248         self.loop = None
 249         if self.msgBus:
 250             self.msgBus.disconnect()