osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from validation import ValidationError, pdu_new_schema, pdu_edit_schema
  24 from base_topic import BaseTopic, EngineException, get_iterable
  25 from osm_im.vnfd import vnfd as vnfd_im
  26 from osm_im.nsd import nsd as nsd_im
  27 from osm_im.nst import nst as nst_im
  28 from pyangbind.lib.serialise import pybindJSONDecoder
  29 import pyangbind.lib.pybindJSON as pybindJSON
  30
  31 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  32
  33
  34 class DescriptorTopic(BaseTopic):
  35
  36     def __init__(self, db, fs, msg):
  37         BaseTopic.__init__(self, db, fs, msg)
  38
  39     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
  40         # 1. validate again with pyangbind
  41         # 1.1. remove internal keys
  42         internal_keys = {}
  43         for k in ("_id", "_admin"):
  44             if k in final_content:
  45                 internal_keys[k] = final_content.pop(k)
  46         serialized = self._validate_input_new(final_content, force)
  47         # 1.2. modify final_content with a serialized version
  48         final_content.clear()
  49         final_content.update(serialized)
  50         # 1.3. restore internal keys
  51         for k, v in internal_keys.items():
  52             final_content[k] = v
  53
  54         if force:
  55             return
  56         # 2. check that this id is not present
  57         if "id" in edit_content:
  58             _filter = self._get_project_filter(session, write=False, show_all=False)
  59             _filter["id"] = final_content["id"]
  60             _filter["_id.neq"] = _id
  61             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  62                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  63                                                                                                final_content["id"]),
  64                                       HTTPStatus.CONFLICT)
  65
  66     @staticmethod
  67     def format_on_new(content, project_id=None, make_public=False):
  68         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  69         content["_admin"]["onboardingState"] = "CREATED"
  70         content["_admin"]["operationalState"] = "DISABLED"
  71         content["_admin"]["usageState"] = "NOT_IN_USE"
  72
  73     def delete(self, session, _id, force=False, dry_run=False):
  74         """
  75         Delete item by its internal _id
  76         :param session: contains the used login username, working project, and admin rights
  77         :param _id: server internal id
  78         :param force: indicates if deletion must be forced in case of conflict
  79         :param dry_run: make checking but do not delete
  80         :return: dictionary with deleted item _id. It raises EngineException on error: not found, conflict, ...
  81         """
  82         # TODO add admin to filter, validate rights
  83         v = BaseTopic.delete(self, session, _id, force, dry_run=True)
  84         if dry_run:
  85             return
  86         v = self.db.del_one(self.topic, {"_id": _id})
  87         self.fs.file_delete(_id, ignore_non_exist=True)
  88         self._send_msg("delete", {"_id": _id})
  89         return v
  90
  91     @staticmethod
  92     def get_one_by_id(db, session, topic, id):
  93         # find owned by this project
  94         _filter = BaseTopic._get_project_filter(session, write=False, show_all=False)
  95         _filter["id"] = id
  96         desc_list = db.get_list(topic, _filter)
  97         if len(desc_list) == 1:
  98             return desc_list[0]
  99         elif len(desc_list) > 1:
 100             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 101                               HTTPStatus.CONFLICT)
 102
 103         # not found any: try to find public
 104         _filter = BaseTopic._get_project_filter(session, write=False, show_all=True)
 105         _filter["id"] = id
 106         desc_list = db.get_list(topic, _filter)
 107         if not desc_list:
 108             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 109         elif len(desc_list) == 1:
 110             return desc_list[0]
 111         else:
 112             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 113                 topic[:-1], id), HTTPStatus.CONFLICT)
 114
 115     def new(self, rollback, session, indata=None, kwargs=None, headers=None, force=False, make_public=False):
 116         """
 117         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 118         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 119         (self.upload_content)
 120         :param rollback: list to append created items at database in case a rollback may to be done
 121         :param session: contains the used login username and working project
 122         :param indata: data to be inserted
 123         :param kwargs: used to override the indata descriptor
 124         :param headers: http request headers
 125         :param force: If True avoid some dependence checks
 126         :param make_public: Make the created descriptor public to all projects
 127         :return: _id: identity of the inserted data.
 128         """
 129
 130         try:
 131             # _remove_envelop
 132             if indata:
 133                 if "userDefinedData" in indata:
 134                     indata = indata['userDefinedData']
 135
 136             # Override descriptor with query string kwargs
 137             self._update_input_with_kwargs(indata, kwargs)
 138             # uncomment when this method is implemented.
 139             # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 140             # indata = DescriptorTopic._validate_input_new(self, indata, force=force)
 141
 142             content = {"_admin": {"userDefinedData": indata}}
 143             self.format_on_new(content, session["project_id"], make_public=make_public)
 144             _id = self.db.create(self.topic, content)
 145             rollback.append({"topic": self.topic, "_id": _id})
 146             return _id
 147         except ValidationError as e:
 148             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 149
 150     def upload_content(self, session, _id, indata, kwargs, headers, force=False):
 151         """
 152         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 153         :param session: session
 154         :param _id : the nsd,vnfd is already created, this is the id
 155         :param indata: http body request
 156         :param kwargs: user query string to override parameters. NOT USED
 157         :param headers:  http request headers
 158         :param force: to be more tolerant with validation
 159         :return: True if package is completely uploaded or False if partial content has been uploded
 160             Raise exception on error
 161         """
 162         # Check that _id exists and it is valid
 163         current_desc = self.show(session, _id)
 164
 165         content_range_text = headers.get("Content-Range")
 166         expected_md5 = headers.get("Content-File-MD5")
 167         compressed = None
 168         content_type = headers.get("Content-Type")
 169         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 170                 "application/zip" in content_type:
 171             compressed = "gzip"
 172         filename = headers.get("Content-Filename")
 173         if not filename:
 174             filename = "package.tar.gz" if compressed else "package"
 175         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 176         file_pkg = None
 177         error_text = ""
 178         try:
 179             if content_range_text:
 180                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 181                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 182                     raise IndexError()
 183                 start = int(content_range[1])
 184                 end = int(content_range[2]) + 1
 185                 total = int(content_range[3])
 186             else:
 187                 start = 0
 188
 189             if start:
 190                 if not self.fs.file_exists(_id, 'dir'):
 191                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 192             else:
 193                 self.fs.file_delete(_id, ignore_non_exist=True)
 194                 self.fs.mkdir(_id)
 195
 196             storage = self.fs.get_params()
 197             storage["folder"] = _id
 198
 199             file_path = (_id, filename)
 200             if self.fs.file_exists(file_path, 'file'):
 201                 file_size = self.fs.file_size(file_path)
 202             else:
 203                 file_size = 0
 204             if file_size != start:
 205                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 206                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 207             file_pkg = self.fs.file_open(file_path, 'a+b')
 208             if isinstance(indata, dict):
 209                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 210                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 211             else:
 212                 indata_len = 0
 213                 while True:
 214                     indata_text = indata.read(4096)
 215                     indata_len += len(indata_text)
 216                     if not indata_text:
 217                         break
 218                     file_pkg.write(indata_text)
 219             if content_range_text:
 220                 if indata_len != end-start:
 221                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 222                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 223                 if end != total:
 224                     # TODO update to UPLOADING
 225                     return False
 226
 227             # PACKAGE UPLOADED
 228             if expected_md5:
 229                 file_pkg.seek(0, 0)
 230                 file_md5 = md5()
 231                 chunk_data = file_pkg.read(1024)
 232                 while chunk_data:
 233                     file_md5.update(chunk_data)
 234                     chunk_data = file_pkg.read(1024)
 235                 if expected_md5 != file_md5.hexdigest():
 236                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 237             file_pkg.seek(0, 0)
 238             if compressed == "gzip":
 239                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 240                 descriptor_file_name = None
 241                 for tarinfo in tar:
 242                     tarname = tarinfo.name
 243                     tarname_path = tarname.split("/")
 244                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 245                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 246                     if len(tarname_path) == 1 and not tarinfo.isdir():
 247                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 248                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 249                         storage["pkg-dir"] = tarname_path[0]
 250                         if len(tarname_path) == 2:
 251                             if descriptor_file_name:
 252                                 raise EngineException(
 253                                     "Found more than one descriptor file at package descriptor tar.gz")
 254                             descriptor_file_name = tarname
 255                 if not descriptor_file_name:
 256                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 257                 storage["descriptor"] = descriptor_file_name
 258                 storage["zipfile"] = filename
 259                 self.fs.file_extract(tar, _id)
 260                 with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
 261                     content = descriptor_file.read()
 262             else:
 263                 content = file_pkg.read()
 264                 storage["descriptor"] = descriptor_file_name = filename
 265
 266             if descriptor_file_name.endswith(".json"):
 267                 error_text = "Invalid json format "
 268                 indata = json.load(content)
 269             else:
 270                 error_text = "Invalid yaml format "
 271                 indata = yaml.load(content)
 272
 273             current_desc["_admin"]["storage"] = storage
 274             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 275             current_desc["_admin"]["operationalState"] = "ENABLED"
 276
 277             indata = self._remove_envelop(indata)
 278
 279             # Override descriptor with query string kwargs
 280             if kwargs:
 281                 self._update_input_with_kwargs(indata, kwargs)
 282             # it will call overrides method at VnfdTopic or NsdTopic
 283             # indata = self._validate_input_edit(indata, force=force)
 284
 285             deep_update_rfc7396(current_desc, indata)
 286             self.check_conflict_on_edit(session, current_desc, indata, _id=_id, force=force)
 287             self.db.replace(self.topic, _id, current_desc)
 288
 289             indata["_id"] = _id
 290             self._send_msg("created", indata)
 291
 292             # TODO if descriptor has changed because kwargs update content and remove cached zip
 293             # TODO if zip is not present creates one
 294             return True
 295
 296         except EngineException:
 297             raise
 298         except IndexError:
 299             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 300                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 301         except IOError as e:
 302             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 303         except tarfile.ReadError as e:
 304             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 305         except (ValueError, yaml.YAMLError) as e:
 306             raise EngineException(error_text + str(e))
 307         except ValidationError as e:
 308             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 309         finally:
 310             if file_pkg:
 311                 file_pkg.close()
 312
 313     def get_file(self, session, _id, path=None, accept_header=None):
 314         """
 315         Return the file content of a vnfd or nsd
 316         :param session: contains the used login username and working project
 317         :param _id: Identity of the vnfd, nsd
 318         :param path: artifact path or "$DESCRIPTOR" or None
 319         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 320         :return: opened file plus Accept format or raises an exception
 321         """
 322         accept_text = accept_zip = False
 323         if accept_header:
 324             if 'text/plain' in accept_header or '*/*' in accept_header:
 325                 accept_text = True
 326             if 'application/zip' in accept_header or '*/*' in accept_header:
 327                 accept_zip = 'application/zip'
 328             elif 'application/gzip' in accept_header:
 329                 accept_zip = 'application/gzip'
 330
 331         if not accept_text and not accept_zip:
 332             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 333                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 334
 335         content = self.show(session, _id)
 336         if content["_admin"]["onboardingState"] != "ONBOARDED":
 337             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 338                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 339                                   http_code=HTTPStatus.CONFLICT)
 340         storage = content["_admin"]["storage"]
 341         if path is not None and path != "$DESCRIPTOR":   # artifacts
 342             if not storage.get('pkg-dir'):
 343                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 344             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 345                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 346                 return folder_content, "text/plain"
 347                 # TODO manage folders in http
 348             else:
 349                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 350                     "application/octet-stream"
 351
 352         # pkgtype   accept  ZIP  TEXT    -> result
 353         # manyfiles         yes  X       -> zip
 354         #                   no   yes     -> error
 355         # onefile           yes  no      -> zip
 356         #                   X    yes     -> text
 357
 358         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 359             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 360         elif storage.get('pkg-dir') and not accept_zip:
 361             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 362                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 363         else:
 364             if not storage.get('zipfile'):
 365                 # TODO generate zipfile if not present
 366                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 367                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 368             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 369
 370     def pyangbind_validation(self, item, data, force=False):
 371         try:
 372             if item == "vnfds":
 373                 myvnfd = vnfd_im()
 374                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 375                                                  path_helper=True, skip_unknown=force)
 376                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 377             elif item == "nsds":
 378                 mynsd = nsd_im()
 379                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 380                                                  path_helper=True, skip_unknown=force)
 381                 out = pybindJSON.dumps(mynsd, mode="ietf")
 382             elif item == "nsts":
 383                 mynst = nst_im()
 384                 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
 385                                                  path_helper=True, skip_unknown=force)
 386                 out = pybindJSON.dumps(mynst, mode="ietf")
 387             else:
 388                 raise EngineException("Not possible to validate '{}' item".format(item),
 389                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 390
 391             desc_out = self._remove_envelop(yaml.safe_load(out))
 392             return desc_out
 393
 394         except Exception as e:
 395             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 396                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 397
 398
 399 class VnfdTopic(DescriptorTopic):
 400     topic = "vnfds"
 401     topic_msg = "vnfd"
 402
 403     def __init__(self, db, fs, msg):
 404         DescriptorTopic.__init__(self, db, fs, msg)
 405
 406     @staticmethod
 407     def _remove_envelop(indata=None):
 408         if not indata:
 409             return {}
 410         clean_indata = indata
 411         if clean_indata.get('vnfd:vnfd-catalog'):
 412             clean_indata = clean_indata['vnfd:vnfd-catalog']
 413         elif clean_indata.get('vnfd-catalog'):
 414             clean_indata = clean_indata['vnfd-catalog']
 415         if clean_indata.get('vnfd'):
 416             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 417                 raise EngineException("'vnfd' must be a list of only one element")
 418             clean_indata = clean_indata['vnfd'][0]
 419         elif clean_indata.get('vnfd:vnfd'):
 420             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 421                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 422             clean_indata = clean_indata['vnfd:vnfd'][0]
 423         return clean_indata
 424
 425     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 426         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 427
 428         # set type of vnfd
 429         contains_pdu = False
 430         contains_vdu = False
 431         for vdu in get_iterable(final_content.get("vdu")):
 432             if vdu.get("pdu-type"):
 433                 contains_pdu = True
 434             else:
 435                 contains_vdu = True
 436         if contains_pdu:
 437             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 438         elif contains_vdu:
 439             final_content["_admin"]["type"] = "vnfd"
 440         # if neither vud nor pdu do not fill type
 441
 442     def check_conflict_on_del(self, session, _id, force=False):
 443         """
 444         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 445         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 446         that uses this vnfd
 447         :param session:
 448         :param _id: vnfd inernal id
 449         :param force: Avoid this checking
 450         :return: None or raises EngineException with the conflict
 451         """
 452         if force:
 453             return
 454         descriptor = self.db.get_one("vnfds", {"_id": _id})
 455         descriptor_id = descriptor.get("id")
 456         if not descriptor_id:  # empty vnfd not uploaded
 457             return
 458
 459         _filter = self._get_project_filter(session, write=False, show_all=False)
 460         # check vnfrs using this vnfd
 461         _filter["vnfd-id"] = _id
 462         if self.db.get_list("vnfrs", _filter):
 463             raise EngineException("There is some VNFR that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 464         del _filter["vnfd-id"]
 465         # check NSD using this VNFD
 466         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 467         if self.db.get_list("nsds", _filter):
 468             raise EngineException("There is soame NSD that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 469
 470     def _validate_input_new(self, indata, force=False):
 471         indata = self.pyangbind_validation("vnfds", indata, force)
 472         # Cross references validation in the descriptor
 473         if indata.get("vdu"):
 474             if not indata.get("mgmt-interface"):
 475                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 476                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 477             if indata["mgmt-interface"].get("cp"):
 478                 for cp in get_iterable(indata.get("connection-point")):
 479                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 480                         break
 481                 else:
 482                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 483                                           .format(indata["mgmt-interface"]["cp"]),
 484                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 485
 486         for vdu in get_iterable(indata.get("vdu")):
 487             for interface in get_iterable(vdu.get("interface")):
 488                 if interface.get("external-connection-point-ref"):
 489                     for cp in get_iterable(indata.get("connection-point")):
 490                         if cp["name"] == interface["external-connection-point-ref"]:
 491                             break
 492                     else:
 493                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 494                                               "must match an existing connection-point"
 495                                               .format(vdu["id"], interface["name"],
 496                                                       interface["external-connection-point-ref"]),
 497                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 498
 499                 elif interface.get("internal-connection-point-ref"):
 500                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 501                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 502                             break
 503                     else:
 504                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 505                                               "must match an existing vdu:internal-connection-point"
 506                                               .format(vdu["id"], interface["name"],
 507                                                       interface["internal-connection-point-ref"]),
 508                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 509         for ivld in get_iterable(indata.get("internal-vld")):
 510             for icp in get_iterable(ivld.get("internal-connection-point")):
 511                 icp_mark = False
 512                 for vdu in get_iterable(indata.get("vdu")):
 513                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 514                         if icp["id-ref"] == internal_cp["id"]:
 515                             icp_mark = True
 516                             break
 517                     if icp_mark:
 518                         break
 519                 else:
 520                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 521                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 522                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 523             if ivld.get("ip-profile-ref"):
 524                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 525                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 526                         break
 527                 else:
 528                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 529                         ivld["id"], ivld["ip-profile-ref"]),
 530                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 531         for mp in get_iterable(indata.get("monitoring-param")):
 532             if mp.get("vdu-monitoring-param"):
 533                 mp_vmp_mark = False
 534                 for vdu in get_iterable(indata.get("vdu")):
 535                     for vmp in get_iterable(vdu.get("monitoring-param")):
 536                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 537                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 538                             mp_vmp_mark = True
 539                             break
 540                     if mp_vmp_mark:
 541                         break
 542                 else:
 543                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 544                                           "defined at vdu[id='{}'] or vdu does not exist"
 545                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 546                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 547                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 548             elif mp.get("vdu-metric"):
 549                 mp_vm_mark = False
 550                 for vdu in get_iterable(indata.get("vdu")):
 551                     if vdu.get("vdu-configuration"):
 552                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 553                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 554                                     mp["vdu-metric"]["vdu-ref"]:
 555                                 mp_vm_mark = True
 556                                 break
 557                         if mp_vm_mark:
 558                             break
 559                 else:
 560                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 561                                           "vdu[id='{}'] or vdu does not exist"
 562                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 563                                                   mp["vdu-metric"]["vdu-ref"]),
 564                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 565
 566         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 567             for sp in get_iterable(sgd.get("scaling-policy")):
 568                 for sc in get_iterable(sp.get("scaling-criteria")):
 569                     for mp in get_iterable(indata.get("monitoring-param")):
 570                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 571                             break
 572                     else:
 573                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 574                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 575                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 576                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 577             for sgd_vdu in get_iterable(sgd.get("vdu")):
 578                 sgd_vdu_mark = False
 579                 for vdu in get_iterable(indata.get("vdu")):
 580                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 581                         sgd_vdu_mark = True
 582                         break
 583                 if sgd_vdu_mark:
 584                     break
 585             else:
 586                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 587                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 588                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 589             for sca in get_iterable(sgd.get("scaling-config-action")):
 590                 if not indata.get("vnf-configuration"):
 591                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 592                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 593                                           .format(sgd["name"]),
 594                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 595                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 596                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 597                         break
 598                 else:
 599                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 600                                           "primitive-name-ref='{}' does not match any "
 601                                           "vnf-configuration:config-primitive:name"
 602                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 603                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 604         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 605         return indata
 606
 607     def _validate_input_edit(self, indata, force=False):
 608         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 609         return indata
 610
 611
 612 class NsdTopic(DescriptorTopic):
 613     topic = "nsds"
 614     topic_msg = "nsd"
 615
 616     def __init__(self, db, fs, msg):
 617         DescriptorTopic.__init__(self, db, fs, msg)
 618
 619     @staticmethod
 620     def _remove_envelop(indata=None):
 621         if not indata:
 622             return {}
 623         clean_indata = indata
 624
 625         if clean_indata.get('nsd:nsd-catalog'):
 626             clean_indata = clean_indata['nsd:nsd-catalog']
 627         elif clean_indata.get('nsd-catalog'):
 628             clean_indata = clean_indata['nsd-catalog']
 629         if clean_indata.get('nsd'):
 630             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 631                 raise EngineException("'nsd' must be a list of only one element")
 632             clean_indata = clean_indata['nsd'][0]
 633         elif clean_indata.get('nsd:nsd'):
 634             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 635                 raise EngineException("'nsd:nsd' must be a list of only one element")
 636             clean_indata = clean_indata['nsd:nsd'][0]
 637         return clean_indata
 638
 639     def _validate_input_new(self, indata, force=False):
 640         indata = self.pyangbind_validation("nsds", indata, force)
 641         # Cross references validation in the descriptor
 642         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 643         for vld in get_iterable(indata.get("vld")):
 644             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 645                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 646                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 647                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 648                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 649                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 650                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 651                                                                  constituent_vnfd["member-vnf-index"],
 652                                                                  constituent_vnfd["vnfd-id-ref"]),
 653                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 654                         break
 655                 else:
 656                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 657                                           "does not match any constituent-vnfd:member-vnf-index"
 658                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 659                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 660         return indata
 661
 662     def _validate_input_edit(self, indata, force=False):
 663         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 664         return indata
 665
 666     def _check_descriptor_dependencies(self, session, descriptor, force=False):
 667         """
 668         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 669         connection points are ok
 670         :param session: client session information
 671         :param descriptor: descriptor to be inserted or edit
 672         :param force: if true skip dependencies checking
 673         :return: None or raises exception
 674         """
 675         if force:
 676             return
 677         member_vnfd_index = {}
 678         if descriptor.get("constituent-vnfd") and not force:
 679             for vnf in descriptor["constituent-vnfd"]:
 680                 vnfd_id = vnf["vnfd-id-ref"]
 681                 filter_q = self._get_project_filter(session, write=False, show_all=True)
 682                 filter_q["id"] = vnfd_id
 683                 vnf_list = self.db.get_list("vnfds", filter_q)
 684                 if not vnf_list:
 685                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 686                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 687                 # elif len(vnf_list) > 1:
 688                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 689                 #                           http_code=HTTPStatus.CONFLICT)
 690                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 691
 692         # Cross references validation in the descriptor and vnfd connection point validation
 693         for vld in get_iterable(descriptor.get("vld")):
 694             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 695                 # look if this vnfd contains this connection point
 696                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 697                 if not vnfd:
 698                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 699                                           "does not match any constituent-vnfd:member-vnf-index"
 700                                           .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
 701                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 702                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 703                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 704                         break
 705                 else:
 706                     raise EngineException(
 707                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 708                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 709                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 710                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 711                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 712
 713     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 714         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 715
 716         self._check_descriptor_dependencies(session, final_content, force)
 717
 718     def check_conflict_on_del(self, session, _id, force=False):
 719         """
 720         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 721         that NSD can be public and be used by other projects.
 722         :param session:
 723         :param _id: vnfd inernal id
 724         :param force: Avoid this checking
 725         :return: None or raises EngineException with the conflict
 726         """
 727         if force:
 728             return
 729         _filter = self._get_project_filter(session, write=False, show_all=False)
 730         _filter["nsdId"] = _id
 731         if self.db.get_list("nsrs", _filter):
 732             raise EngineException("There is some NSR that depends on this NSD", http_code=HTTPStatus.CONFLICT)
 733
 734
 735 class NstTopic(DescriptorTopic):
 736     topic = "nsts"
 737     topic_msg = "nst"
 738
 739     def __init__(self, db, fs, msg):
 740         DescriptorTopic.__init__(self, db, fs, msg)
 741
 742     @staticmethod
 743     def _remove_envelop(indata=None):
 744         if not indata:
 745             return {}
 746         clean_indata = indata
 747
 748         if clean_indata.get('nst'):
 749             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 750                 raise EngineException("'nst' must be a list only one element")
 751             clean_indata = clean_indata['nst'][0]
 752         elif clean_indata.get('nst:nst'):
 753             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
 754                 raise EngineException("'nst:nst' must be a list only one element")
 755             clean_indata = clean_indata['nst:nst'][0]
 756         return clean_indata
 757
 758     def _validate_input_edit(self, indata, force=False):
 759         # TODO validate with pyangbind, serialize
 760         return indata
 761
 762     def _validate_input_new(self, indata, force=False):
 763         indata = self.pyangbind_validation("nsts", indata, force)
 764         return indata.copy()
 765
 766     def _check_descriptor_dependencies(self, session, descriptor):
 767         """
 768         Check that the dependent descriptors exist on a new descriptor or edition
 769         :param session: client session information
 770         :param descriptor: descriptor to be inserted or edit
 771         :return: None or raises exception
 772         """
 773         if not descriptor.get("netslice-subnet"):
 774             return
 775         for nsd in descriptor["netslice-subnet"]:
 776             nsd_id = nsd["nsd-ref"]
 777             filter_q = self._get_project_filter(session, write=False, show_all=True)
 778             filter_q["id"] = nsd_id
 779             if not self.db.get_list("nsds", filter_q):
 780                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
 781                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
 782
 783     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 784         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 785
 786         self._check_descriptor_dependencies(session, final_content)
 787
 788     def check_conflict_on_del(self, session, _id, force=False):
 789         """
 790         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
 791         that NST can be public and be used by other projects.
 792         :param session:
 793         :param _id: nst internal id
 794         :param force: Avoid this checking
 795         :return: None or raises EngineException with the conflict
 796         """
 797         # TODO: Check this method
 798         if force:
 799             return
 800         # Get Network Slice Template from Database
 801         _filter = self._get_project_filter(session, write=False, show_all=False)
 802         _filter["_id"] = _id
 803         nst = self.db.get_one("nsts", _filter)
 804
 805         # Search NSIs using NST via nst-ref
 806         _filter = self._get_project_filter(session, write=False, show_all=False)
 807         _filter["nst-ref"] = nst["id"]
 808         nsis_list = self.db.get_list("nsis", _filter)
 809         for nsi_item in nsis_list:
 810             if nsi_item["_admin"].get("nsiState") != "TERMINATED":
 811                 raise EngineException("There is some NSIS that depends on this NST", http_code=HTTPStatus.CONFLICT)
 812
 813
 814 class PduTopic(BaseTopic):
 815     topic = "pdus"
 816     topic_msg = "pdu"
 817     schema_new = pdu_new_schema
 818     schema_edit = pdu_edit_schema
 819
 820     def __init__(self, db, fs, msg):
 821         BaseTopic.__init__(self, db, fs, msg)
 822
 823     @staticmethod
 824     def format_on_new(content, project_id=None, make_public=False):
 825         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 826         content["_admin"]["onboardingState"] = "CREATED"
 827         content["_admin"]["operationalState"] = "ENABLED"
 828         content["_admin"]["usageState"] = "NOT_IN_USE"
 829
 830     def check_conflict_on_del(self, session, _id, force=False):
 831         if force:
 832             return
 833         # TODO Is it needed to check descriptors _admin.project_read/project_write??
 834         _filter = {"vdur.pdu-id": _id}
 835         if self.db.get_list("vnfrs", _filter):
 836             raise EngineException("There is some NSR that uses this PDU", http_code=HTTPStatus.CONFLICT)