osm_pla/server/server.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Copyright 2020 ArctosLabs Scandinavia AB
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  15 # implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18
  19 import asyncio
  20 import logging
  21 import itertools
  22 from pathlib import Path
  23
  24 import yaml
  25 from osm_common import dbmemory, dbmongo, msglocal, msgkafka
  26
  27 from osm_pla.config.config import Config
  28 from osm_pla.placement.mznplacement import MznPlacementConductor
  29 from osm_pla.placement.mznplacement import NsPlacementDataFactory
  30
  31
  32 class Server:
  33     pil_price_list_file = Path("/placement/pil_price_list.yaml")
  34     vnf_price_list_file = Path("/placement/vnf_price_list.yaml")
  35
  36     def __init__(self, config: Config, loop=None):
  37         self.log = logging.getLogger("pla.server")
  38         self.db = None
  39         self.msgBus = None
  40         self.config = config
  41         self.loop = loop or asyncio.get_event_loop()
  42
  43         try:
  44             if config.get("database", "driver") == "mongo":
  45                 self.db = dbmongo.DbMongo()
  46                 self.db.db_connect(config.get("database"))
  47             elif config.get("database", "driver") == "memory":
  48                 self.db = dbmemory.DbMemory()
  49                 self.db.db_connect(config.get("database"))
  50             else:
  51                 raise Exception(
  52                     "Invalid configuration param '{}' at '[database]':'driver'".format(
  53                         config.get("database", "driver")
  54                     )
  55                 )
  56
  57             if config.get("message", "driver") == "local":
  58                 self.msgBus = msglocal.MsgLocal()
  59             elif config.get("message", "driver") == "kafka":
  60                 self.msgBus = msgkafka.MsgKafka()
  61             else:
  62                 raise Exception(
  63                     "Invalid message bus driver {}".format(
  64                         config.get("message", "driver")
  65                     )
  66                 )
  67             self.msgBus.loop = loop
  68             self.msgBus.connect(config.get("message"))
  69
  70         except Exception as e:
  71             self.log.exception("kafka setup error. Exception: {}".format(e))
  72
  73     def _get_nslcmop(self, nsdlcmop_id):
  74         """
  75         :param nsdlcmop_id:
  76         :return: nslcmop from database corresponding to nslcmop_id
  77         """
  78         db_filter = {"_id": nsdlcmop_id}
  79         nslcmop = self.db.get_one("nslcmops", db_filter)
  80         return nslcmop
  81
  82     def _get_projects(self):
  83         """
  84         :return: project name to project id mapping
  85         """
  86         projects = self.db.get_list("projects")
  87         return {project["_id"]: project["name"] for project in projects}
  88
  89     def _get_nsd(self, nsd_id):
  90         """
  91         :param nsd_id:
  92         :return: nsd from database corresponding to nsd_id
  93         """
  94         db_filter = {"_id": nsd_id}
  95         return self.db.get_one("nsds", db_filter)
  96
  97     def _get_vim_accounts(self, vim_account_ids):
  98         """
  99         :param vim_account_ids: list of VIM account ids
 100         :return: list of vim account entries from database corresponding to list in vim_accounts_id
 101         """
 102         db_filter = {"_id": vim_account_ids}
 103         return self.db.get_list("vim_accounts", db_filter)
 104
 105     def _read_vnf_price_list(self, price_list_file_path):
 106         """
 107         read vnf price list configuration file
 108         :param price_list_file_path:
 109         :return:
 110         """
 111         with open(str(price_list_file_path)) as pl_fd:
 112             price_list = yaml.safe_load_all(pl_fd)
 113             return next(price_list)
 114
 115     def _price_list_with_project(self, price_list):
 116         """
 117         Figure out if this price list is with  project or not.
 118         Note: to handle the unlikely event that a project is called 'prices' we do not simply check if 'prices'
 119         is in the dict keys for a price list sequence but rather go down one step in the nesting
 120         in which we either have
 121         1) 'prices:{vim_url:...}' if prices are also per project, or
 122         2) '{vim_url:...}' if prices are only per vim
 123
 124         :param price_list:
 125         :return: True if project part of price list, else False
 126         """
 127         price_list_entry_keys = set(price_list[0].keys())
 128         price_list_entry_keys.remove("vnfd")
 129         pl_key = price_list_entry_keys.pop()
 130         entry_to_check = price_list[0][pl_key][0].keys()
 131         return True if "prices" in entry_to_check else False
 132
 133     def _get_vnf_price_list(self, price_list_file_path, project_name=None):
 134         """
 135         read vnf price list configuration file, determine its type and reformat content accordingly
 136
 137         :param price_list_file_path:
 138         :param project_name:
 139         :return: dictionary formatted as {'<vnfd>': {'<vim-url>':'<price>'}}
 140         """
 141         price_list_data = self._read_vnf_price_list(price_list_file_path)
 142         if self._price_list_with_project(price_list_data):
 143             res = {}
 144             for i in price_list_data:
 145                 price_data = (
 146                     i[project_name]
 147                     if type(i[project_name]) is dict
 148                     else i[project_name][0]
 149                 )
 150                 res_component = {
 151                     i["vim_name"]: i["price"] for i in price_data["prices"]
 152                 }
 153                 res.update({i["vnfd"]: res_component})
 154             return res
 155         else:
 156             return {
 157                 i["vnfd"]: {i1["vim_name"]: i1["price"] for i1 in i["prices"]}
 158                 for i in price_list_data
 159             }
 160
 161     def _get_pil_info(self, pil_info_file_path):
 162         """
 163         read and return pil information from file
 164         :param pil_info_file_path: Path to pil_info file
 165         :return pil configuration file content as Python object
 166         """
 167         with open(str(pil_info_file_path)) as pil_fd:
 168             data = yaml.safe_load_all(pil_fd)
 169             return next(data)
 170
 171     def _create_vnf_id_maps(self, nsd):
 172         """
 173         map identifier for 'member-vnf-index' in nsd to syntax that is safe for mzn
 174
 175          return tuples with mappings {<adjusted id>: <original id>} and {<original id>: <adjusted id>}
 176         """
 177         # TODO: Change for multiple DF support
 178         ns_df = nsd.get("df", [{}])[0]
 179         next_idx = itertools.count()
 180         member_vnf_index2mzn = {
 181             e["id"]: "VNF" + str(next(next_idx)) for e in ns_df.get("vnf-profile", [])
 182         }
 183
 184         # reverse the name map dictionary, used when the placement result is remapped
 185         mzn_name2member_vnf_index = {v: k for k, v in member_vnf_index2mzn.items()}
 186
 187         return member_vnf_index2mzn, mzn_name2member_vnf_index
 188
 189     async def get_placement(self, nslcmop_id):
 190         """
 191         - Collects and prepares placement information.
 192         - Request placement computation.
 193         - Formats and distribute placement result
 194
 195         Note: exceptions result in empty response message
 196
 197         :param nslcmop_id:
 198         :return:
 199         """
 200         try:
 201             nslcmop = self._get_nslcmop(nslcmop_id)
 202             nsd = self._get_nsd(nslcmop["operationParams"]["nsdId"])
 203             member_vnf_index2mzn, mzn2member_vnf_index = self._create_vnf_id_maps(nsd)
 204             # adjust vnf identifiers
 205             # TODO: Change for multiple DF support
 206             ns_df = nsd.get("df", [{}])[0]
 207             for vnf_profile in ns_df.get("vnf-profile", []):
 208                 vnf_profile["id"] = member_vnf_index2mzn[vnf_profile["id"]]
 209                 for vlc in vnf_profile.get("virtual-link-connectivity", []):
 210                     for ccpd in vlc.get("constituent-cpd-id", []):
 211                         ccpd["constituent-base-element-id"] = member_vnf_index2mzn[
 212                             ccpd["constituent-base-element-id"]
 213                         ]
 214             self.log.info("adjusted nsd: {}".format(nsd))
 215             projects = self._get_projects()
 216             self.log.info("projects: {}".format(projects))
 217             nslcmop_project = nslcmop["_admin"]["projects_read"][0]
 218             self.log.info("nslcmop_project: {}".format(nslcmop_project))
 219             valid_vim_accounts = nslcmop["operationParams"]["validVimAccounts"]
 220             vim_accounts_data = self._get_vim_accounts(valid_vim_accounts)
 221             vims_information = {_["name"]: _["_id"] for _ in vim_accounts_data}
 222             price_list = self._get_vnf_price_list(
 223                 Server.vnf_price_list_file, projects[nslcmop_project]
 224             )
 225             pil_info = self._get_pil_info(Server.pil_price_list_file)
 226             pinnings = nslcmop["operationParams"].get("vnf", [])
 227             # remap member-vnf-index values according to id map
 228             for pinning in pinnings:
 229                 pinning["member-vnf-index"] = member_vnf_index2mzn[
 230                     pinning["member-vnf-index"]
 231                 ]
 232             self.log.info("pinnings: {}".format(pinnings))
 233             order_constraints = nslcmop["operationParams"].get("placement-constraints")
 234             self.log.info("order constraints: {}".format(order_constraints))
 235
 236             nspd = NsPlacementDataFactory(
 237                 vims_information, price_list, nsd, pil_info, pinnings, order_constraints
 238             ).create_ns_placement_data()
 239
 240             vnf_placement = MznPlacementConductor(self.log).do_placement_computation(
 241                 nspd
 242             )
 243
 244         except Exception as e:
 245             # Note: there is no cure for failure so we have a catch-all clause here
 246             self.log.exception("PLA fault. Exception: {}".format(e))
 247             vnf_placement = []
 248         finally:
 249             # remap names in vnf_placement
 250             for e in vnf_placement:
 251                 e["member-vnf-index"] = mzn2member_vnf_index[e["member-vnf-index"]]
 252             await self.msgBus.aiowrite(
 253                 "pla",
 254                 "placement",
 255                 {"placement": {"vnf": vnf_placement, "nslcmopId": nslcmop_id}},
 256             )
 257
 258     def handle_kafka_command(self, topic, command, params):
 259         self.log.info("Kafka msg arrived: {} {} {}".format(topic, command, params))
 260         if topic == "pla" and command == "get_placement":
 261             nslcmop_id = params.get("nslcmopId")
 262             self.loop.create_task(self.get_placement(nslcmop_id))
 263
 264     async def kafka_read(self):
 265         self.log.info("Task kafka_read start")
 266         while True:
 267             try:
 268                 topics = "pla"
 269                 await self.msgBus.aioread(topics, self.loop, self.handle_kafka_command)
 270             except Exception as e:
 271                 self.log.error("kafka read error. Exception: {}".format(e))
 272                 await asyncio.sleep(5, loop=self.loop)
 273
 274     def run(self):
 275         self.loop.run_until_complete(self.kafka_read())
 276         self.loop.close()
 277         self.loop = None
 278         if self.msgBus:
 279             self.msgBus.disconnect()