osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 import tarfile
   4 import yaml
   5 import json
   6 # import logging
   7 from hashlib import md5
   8 from osm_common.dbbase import DbException, deep_update_rfc7396
   9 from http import HTTPStatus
  10 from validation import ValidationError, pdu_new_schema, pdu_edit_schema
  11 from base_topic import BaseTopic, EngineException, get_iterable
  12 from osm_im.vnfd import vnfd as vnfd_im
  13 from osm_im.nsd import nsd as nsd_im
  14 from pyangbind.lib.serialise import pybindJSONDecoder
  15 import pyangbind.lib.pybindJSON as pybindJSON
  16
  17 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  18
  19
  20 class DescriptorTopic(BaseTopic):
  21
  22     def __init__(self, db, fs, msg):
  23         BaseTopic.__init__(self, db, fs, msg)
  24
  25     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
  26         # 1. validate again with pyangbind
  27         # 1.1. remove internal keys
  28         internal_keys = {}
  29         for k in ("_id", "_admin"):
  30             if k in final_content:
  31                 internal_keys[k] = final_content.pop(k)
  32         serialized = self._validate_input_new(final_content, force)
  33         # 1.2. modify final_content with a serialized version
  34         final_content.clear()
  35         final_content.update(serialized)
  36         # 1.3. restore internal keys
  37         for k, v in internal_keys.items():
  38             final_content[k] = v
  39
  40         if force:
  41             return
  42         # 2. check that this id is not present
  43         if "id" in edit_content:
  44             _filter = self._get_project_filter(session, write=False, show_all=False)
  45             _filter["id"] = final_content["id"]
  46             _filter["_id.neq"] = _id
  47             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  48                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  49                                                                                                final_content["id"]),
  50                                       HTTPStatus.CONFLICT)
  51
  52     @staticmethod
  53     def format_on_new(content, project_id=None, make_public=False):
  54         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  55         content["_admin"]["onboardingState"] = "CREATED"
  56         content["_admin"]["operationalState"] = "DISABLED"
  57         content["_admin"]["usageState"] = "NOT_IN_USE"
  58
  59     def delete(self, session, _id, force=False, dry_run=False):
  60         """
  61         Delete item by its internal _id
  62         :param session: contains the used login username, working project, and admin rights
  63         :param _id: server internal id
  64         :param force: indicates if deletion must be forced in case of conflict
  65         :param dry_run: make checking but do not delete
  66         :return: dictionary with deleted item _id. It raises EngineException on error: not found, conflict, ...
  67         """
  68         # TODO add admin to filter, validate rights
  69         v = BaseTopic.delete(self, session, _id, force, dry_run=True)
  70         if dry_run:
  71             return
  72         v = self.db.del_one(self.topic, {"_id": _id})
  73         self.fs.file_delete(_id, ignore_non_exist=True)
  74         self._send_msg("delete", {"_id": _id})
  75         return v
  76
  77     @staticmethod
  78     def get_one_by_id(db, session, topic, id):
  79         # find owned by this project
  80         _filter = BaseTopic._get_project_filter(session, write=False, show_all=False)
  81         _filter["id"] = id
  82         desc_list = db.get_list(topic, _filter)
  83         if len(desc_list) == 1:
  84             return desc_list[0]
  85         elif len(desc_list) > 1:
  86             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
  87                               HTTPStatus.CONFLICT)
  88
  89         # not found any: try to find public
  90         _filter = BaseTopic._get_project_filter(session, write=False, show_all=True)
  91         _filter["id"] = id
  92         desc_list = db.get_list(topic, _filter)
  93         if not desc_list:
  94             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
  95         elif len(desc_list) == 1:
  96             return desc_list[0]
  97         else:
  98             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
  99                 topic[:-1], id), HTTPStatus.CONFLICT)
 100
 101     def new(self, rollback, session, indata=None, kwargs=None, headers=None, force=False, make_public=False):
 102         """
 103         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 104         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 105         (self.upload_content)
 106         :param rollback: list to append created items at database in case a rollback may to be done
 107         :param session: contains the used login username and working project
 108         :param indata: data to be inserted
 109         :param kwargs: used to override the indata descriptor
 110         :param headers: http request headers
 111         :param force: If True avoid some dependence checks
 112         :param make_public: Make the created descriptor public to all projects
 113         :return: _id: identity of the inserted data.
 114         """
 115
 116         try:
 117             # _remove_envelop
 118             if indata:
 119                 if "userDefinedData" in indata:
 120                     indata = indata['userDefinedData']
 121
 122             # Override descriptor with query string kwargs
 123             self._update_input_with_kwargs(indata, kwargs)
 124             # uncomment when this method is implemented.
 125             # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 126             # indata = DescriptorTopic._validate_input_new(self, indata, force=force)
 127
 128             content = {"_admin": {"userDefinedData": indata}}
 129             self.format_on_new(content, session["project_id"], make_public=make_public)
 130             _id = self.db.create(self.topic, content)
 131             rollback.append({"topic": self.topic, "_id": _id})
 132             return _id
 133         except ValidationError as e:
 134             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 135
 136     def upload_content(self, session, _id, indata, kwargs, headers, force=False):
 137         """
 138         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 139         :param session: session
 140         :param _id : the nsd,vnfd is already created, this is the id
 141         :param indata: http body request
 142         :param kwargs: user query string to override parameters. NOT USED
 143         :param headers:  http request headers
 144         :param force: to be more tolerant with validation
 145         :return: True if package is completely uploaded or False if partial content has been uploded
 146             Raise exception on error
 147         """
 148         # Check that _id exists and it is valid
 149         current_desc = self.show(session, _id)
 150
 151         content_range_text = headers.get("Content-Range")
 152         expected_md5 = headers.get("Content-File-MD5")
 153         compressed = None
 154         content_type = headers.get("Content-Type")
 155         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 156                 "application/zip" in content_type:
 157             compressed = "gzip"
 158         filename = headers.get("Content-Filename")
 159         if not filename:
 160             filename = "package.tar.gz" if compressed else "package"
 161         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 162         file_pkg = None
 163         error_text = ""
 164         try:
 165             if content_range_text:
 166                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 167                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 168                     raise IndexError()
 169                 start = int(content_range[1])
 170                 end = int(content_range[2]) + 1
 171                 total = int(content_range[3])
 172             else:
 173                 start = 0
 174
 175             if start:
 176                 if not self.fs.file_exists(_id, 'dir'):
 177                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 178             else:
 179                 self.fs.file_delete(_id, ignore_non_exist=True)
 180                 self.fs.mkdir(_id)
 181
 182             storage = self.fs.get_params()
 183             storage["folder"] = _id
 184
 185             file_path = (_id, filename)
 186             if self.fs.file_exists(file_path, 'file'):
 187                 file_size = self.fs.file_size(file_path)
 188             else:
 189                 file_size = 0
 190             if file_size != start:
 191                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 192                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 193             file_pkg = self.fs.file_open(file_path, 'a+b')
 194             if isinstance(indata, dict):
 195                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 196                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 197             else:
 198                 indata_len = 0
 199                 while True:
 200                     indata_text = indata.read(4096)
 201                     indata_len += len(indata_text)
 202                     if not indata_text:
 203                         break
 204                     file_pkg.write(indata_text)
 205             if content_range_text:
 206                 if indata_len != end-start:
 207                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 208                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 209                 if end != total:
 210                     # TODO update to UPLOADING
 211                     return False
 212
 213             # PACKAGE UPLOADED
 214             if expected_md5:
 215                 file_pkg.seek(0, 0)
 216                 file_md5 = md5()
 217                 chunk_data = file_pkg.read(1024)
 218                 while chunk_data:
 219                     file_md5.update(chunk_data)
 220                     chunk_data = file_pkg.read(1024)
 221                 if expected_md5 != file_md5.hexdigest():
 222                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 223             file_pkg.seek(0, 0)
 224             if compressed == "gzip":
 225                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 226                 descriptor_file_name = None
 227                 for tarinfo in tar:
 228                     tarname = tarinfo.name
 229                     tarname_path = tarname.split("/")
 230                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 231                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 232                     if len(tarname_path) == 1 and not tarinfo.isdir():
 233                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 234                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 235                         storage["pkg-dir"] = tarname_path[0]
 236                         if len(tarname_path) == 2:
 237                             if descriptor_file_name:
 238                                 raise EngineException(
 239                                     "Found more than one descriptor file at package descriptor tar.gz")
 240                             descriptor_file_name = tarname
 241                 if not descriptor_file_name:
 242                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 243                 storage["descriptor"] = descriptor_file_name
 244                 storage["zipfile"] = filename
 245                 self.fs.file_extract(tar, _id)
 246                 with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
 247                     content = descriptor_file.read()
 248             else:
 249                 content = file_pkg.read()
 250                 storage["descriptor"] = descriptor_file_name = filename
 251
 252             if descriptor_file_name.endswith(".json"):
 253                 error_text = "Invalid json format "
 254                 indata = json.load(content)
 255             else:
 256                 error_text = "Invalid yaml format "
 257                 indata = yaml.load(content)
 258
 259             current_desc["_admin"]["storage"] = storage
 260             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 261             current_desc["_admin"]["operationalState"] = "ENABLED"
 262
 263             indata = self._remove_envelop(indata)
 264
 265             # Override descriptor with query string kwargs
 266             if kwargs:
 267                 self._update_input_with_kwargs(indata, kwargs)
 268             # it will call overrides method at VnfdTopic or NsdTopic
 269             # indata = self._validate_input_edit(indata, force=force)
 270
 271             deep_update_rfc7396(current_desc, indata)
 272             self.check_conflict_on_edit(session, current_desc, indata, _id=_id, force=force)
 273             self.db.replace(self.topic, _id, current_desc)
 274
 275             indata["_id"] = _id
 276             self._send_msg("created", indata)
 277
 278             # TODO if descriptor has changed because kwargs update content and remove cached zip
 279             # TODO if zip is not present creates one
 280             return True
 281
 282         except EngineException:
 283             raise
 284         except IndexError:
 285             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 286                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 287         except IOError as e:
 288             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 289         except tarfile.ReadError as e:
 290             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 291         except (ValueError, yaml.YAMLError) as e:
 292             raise EngineException(error_text + str(e))
 293         except ValidationError as e:
 294             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 295         finally:
 296             if file_pkg:
 297                 file_pkg.close()
 298
 299     def get_file(self, session, _id, path=None, accept_header=None):
 300         """
 301         Return the file content of a vnfd or nsd
 302         :param session: contains the used login username and working project
 303         :param _id: Identity of the vnfd, nsd
 304         :param path: artifact path or "$DESCRIPTOR" or None
 305         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 306         :return: opened file plus Accept format or raises an exception
 307         """
 308         accept_text = accept_zip = False
 309         if accept_header:
 310             if 'text/plain' in accept_header or '*/*' in accept_header:
 311                 accept_text = True
 312             if 'application/zip' in accept_header or '*/*' in accept_header:
 313                 accept_zip = 'application/zip'
 314             elif 'application/gzip' in accept_header:
 315                 accept_zip = 'application/gzip'
 316
 317         if not accept_text and not accept_zip:
 318             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 319                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 320
 321         content = self.show(session, _id)
 322         if content["_admin"]["onboardingState"] != "ONBOARDED":
 323             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 324                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 325                                   http_code=HTTPStatus.CONFLICT)
 326         storage = content["_admin"]["storage"]
 327         if path is not None and path != "$DESCRIPTOR":   # artifacts
 328             if not storage.get('pkg-dir'):
 329                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 330             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 331                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 332                 return folder_content, "text/plain"
 333                 # TODO manage folders in http
 334             else:
 335                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 336                     "application/octet-stream"
 337
 338         # pkgtype   accept  ZIP  TEXT    -> result
 339         # manyfiles         yes  X       -> zip
 340         #                   no   yes     -> error
 341         # onefile           yes  no      -> zip
 342         #                   X    yes     -> text
 343
 344         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 345             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 346         elif storage.get('pkg-dir') and not accept_zip:
 347             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 348                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 349         else:
 350             if not storage.get('zipfile'):
 351                 # TODO generate zipfile if not present
 352                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 353                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 354             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 355
 356     def pyangbind_validation(self, item, data, force=False):
 357         try:
 358             if item == "vnfds":
 359                 myvnfd = vnfd_im()
 360                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 361                                                  path_helper=True, skip_unknown=force)
 362                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 363             elif item == "nsds":
 364                 mynsd = nsd_im()
 365                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 366                                                  path_helper=True, skip_unknown=force)
 367                 out = pybindJSON.dumps(mynsd, mode="ietf")
 368             else:
 369                 raise EngineException("Not possible to validate '{}' item".format(item),
 370                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 371
 372             desc_out = self._remove_envelop(yaml.safe_load(out))
 373             return desc_out
 374
 375         except Exception as e:
 376             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 377                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 378
 379
 380 class VnfdTopic(DescriptorTopic):
 381     topic = "vnfds"
 382     topic_msg = "vnfd"
 383
 384     def __init__(self, db, fs, msg):
 385         DescriptorTopic.__init__(self, db, fs, msg)
 386
 387     @staticmethod
 388     def _remove_envelop(indata=None):
 389         if not indata:
 390             return {}
 391         clean_indata = indata
 392         if clean_indata.get('vnfd:vnfd-catalog'):
 393             clean_indata = clean_indata['vnfd:vnfd-catalog']
 394         elif clean_indata.get('vnfd-catalog'):
 395             clean_indata = clean_indata['vnfd-catalog']
 396         if clean_indata.get('vnfd'):
 397             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 398                 raise EngineException("'vnfd' must be a list of only one element")
 399             clean_indata = clean_indata['vnfd'][0]
 400         elif clean_indata.get('vnfd:vnfd'):
 401             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 402                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 403             clean_indata = clean_indata['vnfd:vnfd'][0]
 404         return clean_indata
 405
 406     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 407         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 408
 409         # set type of vnfd
 410         contains_pdu = False
 411         contains_vdu = False
 412         for vdu in get_iterable(final_content.get("vdu")):
 413             if vdu.get("pdu-type"):
 414                 contains_pdu = True
 415             else:
 416                 contains_vdu = True
 417         if contains_pdu:
 418             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 419         elif contains_vdu:
 420             final_content["_admin"]["type"] = "vnfd"
 421         # if neither vud nor pdu do not fill type
 422
 423     def check_conflict_on_del(self, session, _id, force=False):
 424         """
 425         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 426         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 427         that uses this vnfd
 428         :param session:
 429         :param _id: vnfd inernal id
 430         :param force: Avoid this checking
 431         :return: None or raises EngineException with the conflict
 432         """
 433         if force:
 434             return
 435         descriptor = self.db.get_one("vnfds", {"_id": _id})
 436         descriptor_id = descriptor.get("id")
 437         if not descriptor_id:  # empty vnfd not uploaded
 438             return
 439
 440         _filter = self._get_project_filter(session, write=False, show_all=False)
 441         # check vnfrs using this vnfd
 442         _filter["vnfd-id"] = _id
 443         if self.db.get_list("vnfrs", _filter):
 444             raise EngineException("There is some VNFR that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 445         del _filter["vnfd-id"]
 446         # check NSD using this VNFD
 447         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 448         if self.db.get_list("nsds", _filter):
 449             raise EngineException("There is soame NSD that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 450
 451     def _validate_input_new(self, indata, force=False):
 452         indata = self.pyangbind_validation("vnfds", indata, force)
 453         # Cross references validation in the descriptor
 454         if indata.get("vdu"):
 455             if not indata.get("mgmt-interface"):
 456                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 457                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 458             if indata["mgmt-interface"].get("cp"):
 459                 for cp in get_iterable(indata.get("connection-point")):
 460                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 461                         break
 462                 else:
 463                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 464                                           .format(indata["mgmt-interface"]["cp"]),
 465                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 466
 467         for vdu in get_iterable(indata.get("vdu")):
 468             for interface in get_iterable(vdu.get("interface")):
 469                 if interface.get("external-connection-point-ref"):
 470                     for cp in get_iterable(indata.get("connection-point")):
 471                         if cp["name"] == interface["external-connection-point-ref"]:
 472                             break
 473                     else:
 474                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 475                                               "must match an existing connection-point"
 476                                               .format(vdu["id"], interface["name"],
 477                                                       interface["external-connection-point-ref"]),
 478                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 479
 480                 elif interface.get("internal-connection-point-ref"):
 481                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 482                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 483                             break
 484                     else:
 485                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 486                                               "must match an existing vdu:internal-connection-point"
 487                                               .format(vdu["id"], interface["name"],
 488                                                       interface["internal-connection-point-ref"]),
 489                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 490         for ivld in get_iterable(indata.get("internal-vld")):
 491             for icp in get_iterable(ivld.get("internal-connection-point")):
 492                 icp_mark = False
 493                 for vdu in get_iterable(indata.get("vdu")):
 494                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 495                         if icp["id-ref"] == internal_cp["id"]:
 496                             icp_mark = True
 497                             break
 498                     if icp_mark:
 499                         break
 500                 else:
 501                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 502                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 503                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 504             if ivld.get("ip-profile-ref"):
 505                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 506                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 507                         break
 508                 else:
 509                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 510                         ivld["id"], ivld["ip-profile-ref"]),
 511                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 512         for mp in get_iterable(indata.get("monitoring-param")):
 513             if mp.get("vdu-monitoring-param"):
 514                 mp_vmp_mark = False
 515                 for vdu in get_iterable(indata.get("vdu")):
 516                     for vmp in get_iterable(vdu.get("monitoring-param")):
 517                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 518                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 519                             mp_vmp_mark = True
 520                             break
 521                     if mp_vmp_mark:
 522                         break
 523                 else:
 524                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 525                                           "defined at vdu[id='{}'] or vdu does not exist"
 526                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 527                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 528                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 529             elif mp.get("vdu-metric"):
 530                 mp_vm_mark = False
 531                 for vdu in get_iterable(indata.get("vdu")):
 532                     if vdu.get("vdu-configuration"):
 533                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 534                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 535                                     mp["vdu-metric"]["vdu-ref"]:
 536                                 mp_vm_mark = True
 537                                 break
 538                         if mp_vm_mark:
 539                             break
 540                 else:
 541                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 542                                           "vdu[id='{}'] or vdu does not exist"
 543                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 544                                                   mp["vdu-metric"]["vdu-ref"]),
 545                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 546
 547         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 548             for sp in get_iterable(sgd.get("scaling-policy")):
 549                 for sc in get_iterable(sp.get("scaling-criteria")):
 550                     for mp in get_iterable(indata.get("monitoring-param")):
 551                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 552                             break
 553                     else:
 554                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 555                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 556                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 557                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 558             for sgd_vdu in get_iterable(sgd.get("vdu")):
 559                 sgd_vdu_mark = False
 560                 for vdu in get_iterable(indata.get("vdu")):
 561                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 562                         sgd_vdu_mark = True
 563                         break
 564                 if sgd_vdu_mark:
 565                     break
 566             else:
 567                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 568                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 569                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 570             for sca in get_iterable(sgd.get("scaling-config-action")):
 571                 if not indata.get("vnf-configuration"):
 572                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 573                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 574                                           .format(sgd["name"]),
 575                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 576                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 577                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 578                         break
 579                 else:
 580                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 581                                           "primitive-name-ref='{}' does not match any "
 582                                           "vnf-configuration:config-primitive:name"
 583                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 584                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 585         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 586         return indata
 587
 588     def _validate_input_edit(self, indata, force=False):
 589         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 590         return indata
 591
 592
 593 class NsdTopic(DescriptorTopic):
 594     topic = "nsds"
 595     topic_msg = "nsd"
 596
 597     def __init__(self, db, fs, msg):
 598         DescriptorTopic.__init__(self, db, fs, msg)
 599
 600     @staticmethod
 601     def _remove_envelop(indata=None):
 602         if not indata:
 603             return {}
 604         clean_indata = indata
 605
 606         if clean_indata.get('nsd:nsd-catalog'):
 607             clean_indata = clean_indata['nsd:nsd-catalog']
 608         elif clean_indata.get('nsd-catalog'):
 609             clean_indata = clean_indata['nsd-catalog']
 610         if clean_indata.get('nsd'):
 611             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 612                 raise EngineException("'nsd' must be a list of only one element")
 613             clean_indata = clean_indata['nsd'][0]
 614         elif clean_indata.get('nsd:nsd'):
 615             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 616                 raise EngineException("'nsd:nsd' must be a list of only one element")
 617             clean_indata = clean_indata['nsd:nsd'][0]
 618         return clean_indata
 619
 620     def _validate_input_new(self, indata, force=False):
 621         indata = self.pyangbind_validation("nsds", indata, force)
 622         # Cross references validation in the descriptor
 623         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 624         for vld in get_iterable(indata.get("vld")):
 625             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 626                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 627                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 628                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 629                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 630                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 631                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 632                                                                  constituent_vnfd["member-vnf-index"],
 633                                                                  constituent_vnfd["vnfd-id-ref"]),
 634                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 635                         break
 636                 else:
 637                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 638                                           "does not match any constituent-vnfd:member-vnf-index"
 639                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 640                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 641         return indata
 642
 643     def _validate_input_edit(self, indata, force=False):
 644         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 645         return indata
 646
 647     def _check_descriptor_dependencies(self, session, descriptor, force=False):
 648         """
 649         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 650         connection points are ok
 651         :param session: client session information
 652         :param descriptor: descriptor to be inserted or edit
 653         :param force: if true skip dependencies checking
 654         :return: None or raises exception
 655         """
 656         if force:
 657             return
 658         member_vnfd_index = {}
 659         if descriptor.get("constituent-vnfd") and not force:
 660             for vnf in descriptor["constituent-vnfd"]:
 661                 vnfd_id = vnf["vnfd-id-ref"]
 662                 filter_q = self._get_project_filter(session, write=False, show_all=True)
 663                 filter_q["id"] = vnfd_id
 664                 vnf_list = self.db.get_list("vnfds", filter_q)
 665                 if not vnf_list:
 666                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 667                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 668                 # elif len(vnf_list) > 1:
 669                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 670                 #                           http_code=HTTPStatus.CONFLICT)
 671                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 672
 673         # Cross references validation in the descriptor and vnfd connection point validation
 674         for vld in get_iterable(descriptor.get("vld")):
 675             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 676                 # look if this vnfd contains this connection point
 677                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 678                 if not vnfd:
 679                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 680                                           "does not match any constituent-vnfd:member-vnf-index"
 681                                           .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
 682                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 683                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 684                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 685                         break
 686                 else:
 687                     raise EngineException(
 688                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 689                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 690                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 691                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 692                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 693
 694     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 695         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 696
 697         self._check_descriptor_dependencies(session, final_content, force)
 698
 699     def check_conflict_on_del(self, session, _id, force=False):
 700         """
 701         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 702         that NSD can be public and be used by other projects.
 703         :param session:
 704         :param _id: vnfd inernal id
 705         :param force: Avoid this checking
 706         :return: None or raises EngineException with the conflict
 707         """
 708         if force:
 709             return
 710         _filter = self._get_project_filter(session, write=False, show_all=False)
 711         _filter["nsdId"] = _id
 712         if self.db.get_list("nsrs", _filter):
 713             raise EngineException("There is some NSR that depends on this NSD", http_code=HTTPStatus.CONFLICT)
 714
 715
 716 class NstTopic(DescriptorTopic):
 717     topic = "nsts"
 718     topic_msg = "nst"
 719
 720     def __init__(self, db, fs, msg):
 721         DescriptorTopic.__init__(self, db, fs, msg)
 722
 723     @staticmethod
 724     def _remove_envelop(indata=None):
 725         if not indata:
 726             return {}
 727         clean_indata = indata
 728
 729         if clean_indata.get('nst:nst'):
 730             clean_indata = clean_indata['nst:nst']
 731         elif clean_indata.get('nst'):
 732             clean_indata = clean_indata['nst']
 733         if clean_indata.get('nst'):
 734             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 735                 raise EngineException("'nst' must be a list only one element")
 736             clean_indata = clean_indata['nst'][0]
 737         return clean_indata
 738
 739     def _validate_input_edit(self, indata, force=False):
 740         # TODO validate with pyangbind, serialize
 741         return indata
 742
 743     def _check_descriptor_dependencies(self, session, descriptor):
 744         """
 745         Check that the dependent descriptors exist on a new descriptor or edition
 746         :param session: client session information
 747         :param descriptor: descriptor to be inserted or edit
 748         :return: None or raises exception
 749         """
 750         if not descriptor.get("netslice-subnet"):
 751             return
 752         for nsd in descriptor["netslice-subnet"]:
 753             nsd_id = nsd["nsd-ref"]
 754             filter_q = self._get_project_filter(session, write=False, show_all=True)
 755             filter_q["id"] = nsd_id
 756             if not self.db.get_list("nsds", filter_q):
 757                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
 758                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
 759
 760     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 761         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 762
 763         self._check_descriptor_dependencies(session, final_content)
 764
 765     def check_conflict_on_del(self, session, _id, force=False):
 766         """
 767         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
 768         that NST can be public and be used by other projects.
 769         :param session:
 770         :param _id: nst internal id
 771         :param force: Avoid this checking
 772         :return: None or raises EngineException with the conflict
 773         """
 774         # TODO: Check this method
 775         if force:
 776             return
 777         # Get Network Slice Template from Database
 778         _filter = self._get_project_filter(session, write=False, show_all=False)
 779         _filter["_id"] = _id
 780         nst = self.db.get_one("nst", _filter)
 781
 782         # Search NSIs using NST via nst-ref
 783         _filter = self._get_project_filter(session, write=False, show_all=False)
 784         _filter["nst-ref"] = nst["id"]
 785         if self.db.get_list("nsis", _filter):
 786             raise EngineException("There is some NSIS that depends on this NST", http_code=HTTPStatus.CONFLICT)
 787
 788
 789 class PduTopic(BaseTopic):
 790     topic = "pdus"
 791     topic_msg = "pdu"
 792     schema_new = pdu_new_schema
 793     schema_edit = pdu_edit_schema
 794
 795     def __init__(self, db, fs, msg):
 796         BaseTopic.__init__(self, db, fs, msg)
 797
 798     @staticmethod
 799     def format_on_new(content, project_id=None, make_public=False):
 800         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 801         content["_admin"]["onboardingState"] = "CREATED"
 802         content["_admin"]["operationalState"] = "ENABLED"
 803         content["_admin"]["usageState"] = "NOT_IN_USE"
 804
 805     def check_conflict_on_del(self, session, _id, force=False):
 806         if force:
 807             return
 808         # TODO Is it needed to check descriptors _admin.project_read/project_write??
 809         _filter = {"vdur.pdu-id": _id}
 810         if self.db.get_list("vnfrs", _filter):
 811             raise EngineException("There is some NSR that uses this PDU", http_code=HTTPStatus.CONFLICT)