osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from validation import ValidationError, pdu_new_schema, pdu_edit_schema
  24 from base_topic import BaseTopic, EngineException, get_iterable
  25 from osm_im.vnfd import vnfd as vnfd_im
  26 from osm_im.nsd import nsd as nsd_im
  27 from pyangbind.lib.serialise import pybindJSONDecoder
  28 import pyangbind.lib.pybindJSON as pybindJSON
  29
  30 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  31
  32
  33 class DescriptorTopic(BaseTopic):
  34
  35     def __init__(self, db, fs, msg):
  36         BaseTopic.__init__(self, db, fs, msg)
  37
  38     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
  39         # 1. validate again with pyangbind
  40         # 1.1. remove internal keys
  41         internal_keys = {}
  42         for k in ("_id", "_admin"):
  43             if k in final_content:
  44                 internal_keys[k] = final_content.pop(k)
  45         serialized = self._validate_input_new(final_content, force)
  46         # 1.2. modify final_content with a serialized version
  47         final_content.clear()
  48         final_content.update(serialized)
  49         # 1.3. restore internal keys
  50         for k, v in internal_keys.items():
  51             final_content[k] = v
  52
  53         if force:
  54             return
  55         # 2. check that this id is not present
  56         if "id" in edit_content:
  57             _filter = self._get_project_filter(session, write=False, show_all=False)
  58             _filter["id"] = final_content["id"]
  59             _filter["_id.neq"] = _id
  60             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  61                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  62                                                                                                final_content["id"]),
  63                                       HTTPStatus.CONFLICT)
  64
  65     @staticmethod
  66     def format_on_new(content, project_id=None, make_public=False):
  67         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  68         content["_admin"]["onboardingState"] = "CREATED"
  69         content["_admin"]["operationalState"] = "DISABLED"
  70         content["_admin"]["usageState"] = "NOT_IN_USE"
  71
  72     def delete(self, session, _id, force=False, dry_run=False):
  73         """
  74         Delete item by its internal _id
  75         :param session: contains the used login username, working project, and admin rights
  76         :param _id: server internal id
  77         :param force: indicates if deletion must be forced in case of conflict
  78         :param dry_run: make checking but do not delete
  79         :return: dictionary with deleted item _id. It raises EngineException on error: not found, conflict, ...
  80         """
  81         # TODO add admin to filter, validate rights
  82         v = BaseTopic.delete(self, session, _id, force, dry_run=True)
  83         if dry_run:
  84             return
  85         v = self.db.del_one(self.topic, {"_id": _id})
  86         self.fs.file_delete(_id, ignore_non_exist=True)
  87         self._send_msg("delete", {"_id": _id})
  88         return v
  89
  90     @staticmethod
  91     def get_one_by_id(db, session, topic, id):
  92         # find owned by this project
  93         _filter = BaseTopic._get_project_filter(session, write=False, show_all=False)
  94         _filter["id"] = id
  95         desc_list = db.get_list(topic, _filter)
  96         if len(desc_list) == 1:
  97             return desc_list[0]
  98         elif len(desc_list) > 1:
  99             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 100                               HTTPStatus.CONFLICT)
 101
 102         # not found any: try to find public
 103         _filter = BaseTopic._get_project_filter(session, write=False, show_all=True)
 104         _filter["id"] = id
 105         desc_list = db.get_list(topic, _filter)
 106         if not desc_list:
 107             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 108         elif len(desc_list) == 1:
 109             return desc_list[0]
 110         else:
 111             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 112                 topic[:-1], id), HTTPStatus.CONFLICT)
 113
 114     def new(self, rollback, session, indata=None, kwargs=None, headers=None, force=False, make_public=False):
 115         """
 116         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 117         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 118         (self.upload_content)
 119         :param rollback: list to append created items at database in case a rollback may to be done
 120         :param session: contains the used login username and working project
 121         :param indata: data to be inserted
 122         :param kwargs: used to override the indata descriptor
 123         :param headers: http request headers
 124         :param force: If True avoid some dependence checks
 125         :param make_public: Make the created descriptor public to all projects
 126         :return: _id: identity of the inserted data.
 127         """
 128
 129         try:
 130             # _remove_envelop
 131             if indata:
 132                 if "userDefinedData" in indata:
 133                     indata = indata['userDefinedData']
 134
 135             # Override descriptor with query string kwargs
 136             self._update_input_with_kwargs(indata, kwargs)
 137             # uncomment when this method is implemented.
 138             # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 139             # indata = DescriptorTopic._validate_input_new(self, indata, force=force)
 140
 141             content = {"_admin": {"userDefinedData": indata}}
 142             self.format_on_new(content, session["project_id"], make_public=make_public)
 143             _id = self.db.create(self.topic, content)
 144             rollback.append({"topic": self.topic, "_id": _id})
 145             return _id
 146         except ValidationError as e:
 147             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 148
 149     def upload_content(self, session, _id, indata, kwargs, headers, force=False):
 150         """
 151         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 152         :param session: session
 153         :param _id : the nsd,vnfd is already created, this is the id
 154         :param indata: http body request
 155         :param kwargs: user query string to override parameters. NOT USED
 156         :param headers:  http request headers
 157         :param force: to be more tolerant with validation
 158         :return: True if package is completely uploaded or False if partial content has been uploded
 159             Raise exception on error
 160         """
 161         # Check that _id exists and it is valid
 162         current_desc = self.show(session, _id)
 163
 164         content_range_text = headers.get("Content-Range")
 165         expected_md5 = headers.get("Content-File-MD5")
 166         compressed = None
 167         content_type = headers.get("Content-Type")
 168         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 169                 "application/zip" in content_type:
 170             compressed = "gzip"
 171         filename = headers.get("Content-Filename")
 172         if not filename:
 173             filename = "package.tar.gz" if compressed else "package"
 174         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 175         file_pkg = None
 176         error_text = ""
 177         try:
 178             if content_range_text:
 179                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 180                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 181                     raise IndexError()
 182                 start = int(content_range[1])
 183                 end = int(content_range[2]) + 1
 184                 total = int(content_range[3])
 185             else:
 186                 start = 0
 187
 188             if start:
 189                 if not self.fs.file_exists(_id, 'dir'):
 190                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 191             else:
 192                 self.fs.file_delete(_id, ignore_non_exist=True)
 193                 self.fs.mkdir(_id)
 194
 195             storage = self.fs.get_params()
 196             storage["folder"] = _id
 197
 198             file_path = (_id, filename)
 199             if self.fs.file_exists(file_path, 'file'):
 200                 file_size = self.fs.file_size(file_path)
 201             else:
 202                 file_size = 0
 203             if file_size != start:
 204                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 205                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 206             file_pkg = self.fs.file_open(file_path, 'a+b')
 207             if isinstance(indata, dict):
 208                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 209                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 210             else:
 211                 indata_len = 0
 212                 while True:
 213                     indata_text = indata.read(4096)
 214                     indata_len += len(indata_text)
 215                     if not indata_text:
 216                         break
 217                     file_pkg.write(indata_text)
 218             if content_range_text:
 219                 if indata_len != end-start:
 220                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 221                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 222                 if end != total:
 223                     # TODO update to UPLOADING
 224                     return False
 225
 226             # PACKAGE UPLOADED
 227             if expected_md5:
 228                 file_pkg.seek(0, 0)
 229                 file_md5 = md5()
 230                 chunk_data = file_pkg.read(1024)
 231                 while chunk_data:
 232                     file_md5.update(chunk_data)
 233                     chunk_data = file_pkg.read(1024)
 234                 if expected_md5 != file_md5.hexdigest():
 235                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 236             file_pkg.seek(0, 0)
 237             if compressed == "gzip":
 238                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 239                 descriptor_file_name = None
 240                 for tarinfo in tar:
 241                     tarname = tarinfo.name
 242                     tarname_path = tarname.split("/")
 243                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 244                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 245                     if len(tarname_path) == 1 and not tarinfo.isdir():
 246                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 247                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 248                         storage["pkg-dir"] = tarname_path[0]
 249                         if len(tarname_path) == 2:
 250                             if descriptor_file_name:
 251                                 raise EngineException(
 252                                     "Found more than one descriptor file at package descriptor tar.gz")
 253                             descriptor_file_name = tarname
 254                 if not descriptor_file_name:
 255                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 256                 storage["descriptor"] = descriptor_file_name
 257                 storage["zipfile"] = filename
 258                 self.fs.file_extract(tar, _id)
 259                 with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
 260                     content = descriptor_file.read()
 261             else:
 262                 content = file_pkg.read()
 263                 storage["descriptor"] = descriptor_file_name = filename
 264
 265             if descriptor_file_name.endswith(".json"):
 266                 error_text = "Invalid json format "
 267                 indata = json.load(content)
 268             else:
 269                 error_text = "Invalid yaml format "
 270                 indata = yaml.load(content)
 271
 272             current_desc["_admin"]["storage"] = storage
 273             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 274             current_desc["_admin"]["operationalState"] = "ENABLED"
 275
 276             indata = self._remove_envelop(indata)
 277
 278             # Override descriptor with query string kwargs
 279             if kwargs:
 280                 self._update_input_with_kwargs(indata, kwargs)
 281             # it will call overrides method at VnfdTopic or NsdTopic
 282             # indata = self._validate_input_edit(indata, force=force)
 283
 284             deep_update_rfc7396(current_desc, indata)
 285             self.check_conflict_on_edit(session, current_desc, indata, _id=_id, force=force)
 286             self.db.replace(self.topic, _id, current_desc)
 287
 288             indata["_id"] = _id
 289             self._send_msg("created", indata)
 290
 291             # TODO if descriptor has changed because kwargs update content and remove cached zip
 292             # TODO if zip is not present creates one
 293             return True
 294
 295         except EngineException:
 296             raise
 297         except IndexError:
 298             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 299                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 300         except IOError as e:
 301             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 302         except tarfile.ReadError as e:
 303             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 304         except (ValueError, yaml.YAMLError) as e:
 305             raise EngineException(error_text + str(e))
 306         except ValidationError as e:
 307             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 308         finally:
 309             if file_pkg:
 310                 file_pkg.close()
 311
 312     def get_file(self, session, _id, path=None, accept_header=None):
 313         """
 314         Return the file content of a vnfd or nsd
 315         :param session: contains the used login username and working project
 316         :param _id: Identity of the vnfd, nsd
 317         :param path: artifact path or "$DESCRIPTOR" or None
 318         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 319         :return: opened file plus Accept format or raises an exception
 320         """
 321         accept_text = accept_zip = False
 322         if accept_header:
 323             if 'text/plain' in accept_header or '*/*' in accept_header:
 324                 accept_text = True
 325             if 'application/zip' in accept_header or '*/*' in accept_header:
 326                 accept_zip = 'application/zip'
 327             elif 'application/gzip' in accept_header:
 328                 accept_zip = 'application/gzip'
 329
 330         if not accept_text and not accept_zip:
 331             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 332                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 333
 334         content = self.show(session, _id)
 335         if content["_admin"]["onboardingState"] != "ONBOARDED":
 336             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 337                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 338                                   http_code=HTTPStatus.CONFLICT)
 339         storage = content["_admin"]["storage"]
 340         if path is not None and path != "$DESCRIPTOR":   # artifacts
 341             if not storage.get('pkg-dir'):
 342                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 343             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 344                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 345                 return folder_content, "text/plain"
 346                 # TODO manage folders in http
 347             else:
 348                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 349                     "application/octet-stream"
 350
 351         # pkgtype   accept  ZIP  TEXT    -> result
 352         # manyfiles         yes  X       -> zip
 353         #                   no   yes     -> error
 354         # onefile           yes  no      -> zip
 355         #                   X    yes     -> text
 356
 357         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 358             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 359         elif storage.get('pkg-dir') and not accept_zip:
 360             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 361                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 362         else:
 363             if not storage.get('zipfile'):
 364                 # TODO generate zipfile if not present
 365                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 366                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 367             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 368
 369     def pyangbind_validation(self, item, data, force=False):
 370         try:
 371             if item == "vnfds":
 372                 myvnfd = vnfd_im()
 373                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 374                                                  path_helper=True, skip_unknown=force)
 375                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 376             elif item == "nsds":
 377                 mynsd = nsd_im()
 378                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 379                                                  path_helper=True, skip_unknown=force)
 380                 out = pybindJSON.dumps(mynsd, mode="ietf")
 381             else:
 382                 raise EngineException("Not possible to validate '{}' item".format(item),
 383                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 384
 385             desc_out = self._remove_envelop(yaml.safe_load(out))
 386             return desc_out
 387
 388         except Exception as e:
 389             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 390                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 391
 392
 393 class VnfdTopic(DescriptorTopic):
 394     topic = "vnfds"
 395     topic_msg = "vnfd"
 396
 397     def __init__(self, db, fs, msg):
 398         DescriptorTopic.__init__(self, db, fs, msg)
 399
 400     @staticmethod
 401     def _remove_envelop(indata=None):
 402         if not indata:
 403             return {}
 404         clean_indata = indata
 405         if clean_indata.get('vnfd:vnfd-catalog'):
 406             clean_indata = clean_indata['vnfd:vnfd-catalog']
 407         elif clean_indata.get('vnfd-catalog'):
 408             clean_indata = clean_indata['vnfd-catalog']
 409         if clean_indata.get('vnfd'):
 410             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 411                 raise EngineException("'vnfd' must be a list of only one element")
 412             clean_indata = clean_indata['vnfd'][0]
 413         elif clean_indata.get('vnfd:vnfd'):
 414             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 415                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 416             clean_indata = clean_indata['vnfd:vnfd'][0]
 417         return clean_indata
 418
 419     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 420         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 421
 422         # set type of vnfd
 423         contains_pdu = False
 424         contains_vdu = False
 425         for vdu in get_iterable(final_content.get("vdu")):
 426             if vdu.get("pdu-type"):
 427                 contains_pdu = True
 428             else:
 429                 contains_vdu = True
 430         if contains_pdu:
 431             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 432         elif contains_vdu:
 433             final_content["_admin"]["type"] = "vnfd"
 434         # if neither vud nor pdu do not fill type
 435
 436     def check_conflict_on_del(self, session, _id, force=False):
 437         """
 438         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 439         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 440         that uses this vnfd
 441         :param session:
 442         :param _id: vnfd inernal id
 443         :param force: Avoid this checking
 444         :return: None or raises EngineException with the conflict
 445         """
 446         if force:
 447             return
 448         descriptor = self.db.get_one("vnfds", {"_id": _id})
 449         descriptor_id = descriptor.get("id")
 450         if not descriptor_id:  # empty vnfd not uploaded
 451             return
 452
 453         _filter = self._get_project_filter(session, write=False, show_all=False)
 454         # check vnfrs using this vnfd
 455         _filter["vnfd-id"] = _id
 456         if self.db.get_list("vnfrs", _filter):
 457             raise EngineException("There is some VNFR that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 458         del _filter["vnfd-id"]
 459         # check NSD using this VNFD
 460         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 461         if self.db.get_list("nsds", _filter):
 462             raise EngineException("There is soame NSD that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 463
 464     def _validate_input_new(self, indata, force=False):
 465         indata = self.pyangbind_validation("vnfds", indata, force)
 466         # Cross references validation in the descriptor
 467         if indata.get("vdu"):
 468             if not indata.get("mgmt-interface"):
 469                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 470                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 471             if indata["mgmt-interface"].get("cp"):
 472                 for cp in get_iterable(indata.get("connection-point")):
 473                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 474                         break
 475                 else:
 476                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 477                                           .format(indata["mgmt-interface"]["cp"]),
 478                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 479
 480         for vdu in get_iterable(indata.get("vdu")):
 481             for interface in get_iterable(vdu.get("interface")):
 482                 if interface.get("external-connection-point-ref"):
 483                     for cp in get_iterable(indata.get("connection-point")):
 484                         if cp["name"] == interface["external-connection-point-ref"]:
 485                             break
 486                     else:
 487                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 488                                               "must match an existing connection-point"
 489                                               .format(vdu["id"], interface["name"],
 490                                                       interface["external-connection-point-ref"]),
 491                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 492
 493                 elif interface.get("internal-connection-point-ref"):
 494                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 495                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 496                             break
 497                     else:
 498                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 499                                               "must match an existing vdu:internal-connection-point"
 500                                               .format(vdu["id"], interface["name"],
 501                                                       interface["internal-connection-point-ref"]),
 502                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 503         for ivld in get_iterable(indata.get("internal-vld")):
 504             for icp in get_iterable(ivld.get("internal-connection-point")):
 505                 icp_mark = False
 506                 for vdu in get_iterable(indata.get("vdu")):
 507                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 508                         if icp["id-ref"] == internal_cp["id"]:
 509                             icp_mark = True
 510                             break
 511                     if icp_mark:
 512                         break
 513                 else:
 514                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 515                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 516                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 517             if ivld.get("ip-profile-ref"):
 518                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 519                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 520                         break
 521                 else:
 522                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 523                         ivld["id"], ivld["ip-profile-ref"]),
 524                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 525         for mp in get_iterable(indata.get("monitoring-param")):
 526             if mp.get("vdu-monitoring-param"):
 527                 mp_vmp_mark = False
 528                 for vdu in get_iterable(indata.get("vdu")):
 529                     for vmp in get_iterable(vdu.get("monitoring-param")):
 530                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 531                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 532                             mp_vmp_mark = True
 533                             break
 534                     if mp_vmp_mark:
 535                         break
 536                 else:
 537                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 538                                           "defined at vdu[id='{}'] or vdu does not exist"
 539                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 540                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 541                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 542             elif mp.get("vdu-metric"):
 543                 mp_vm_mark = False
 544                 for vdu in get_iterable(indata.get("vdu")):
 545                     if vdu.get("vdu-configuration"):
 546                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 547                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 548                                     mp["vdu-metric"]["vdu-ref"]:
 549                                 mp_vm_mark = True
 550                                 break
 551                         if mp_vm_mark:
 552                             break
 553                 else:
 554                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 555                                           "vdu[id='{}'] or vdu does not exist"
 556                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 557                                                   mp["vdu-metric"]["vdu-ref"]),
 558                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 559
 560         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 561             for sp in get_iterable(sgd.get("scaling-policy")):
 562                 for sc in get_iterable(sp.get("scaling-criteria")):
 563                     for mp in get_iterable(indata.get("monitoring-param")):
 564                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 565                             break
 566                     else:
 567                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 568                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 569                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 570                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 571             for sgd_vdu in get_iterable(sgd.get("vdu")):
 572                 sgd_vdu_mark = False
 573                 for vdu in get_iterable(indata.get("vdu")):
 574                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 575                         sgd_vdu_mark = True
 576                         break
 577                 if sgd_vdu_mark:
 578                     break
 579             else:
 580                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 581                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 582                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 583             for sca in get_iterable(sgd.get("scaling-config-action")):
 584                 if not indata.get("vnf-configuration"):
 585                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 586                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 587                                           .format(sgd["name"]),
 588                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 589                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 590                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 591                         break
 592                 else:
 593                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 594                                           "primitive-name-ref='{}' does not match any "
 595                                           "vnf-configuration:config-primitive:name"
 596                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 597                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 598         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 599         return indata
 600
 601     def _validate_input_edit(self, indata, force=False):
 602         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 603         return indata
 604
 605
 606 class NsdTopic(DescriptorTopic):
 607     topic = "nsds"
 608     topic_msg = "nsd"
 609
 610     def __init__(self, db, fs, msg):
 611         DescriptorTopic.__init__(self, db, fs, msg)
 612
 613     @staticmethod
 614     def _remove_envelop(indata=None):
 615         if not indata:
 616             return {}
 617         clean_indata = indata
 618
 619         if clean_indata.get('nsd:nsd-catalog'):
 620             clean_indata = clean_indata['nsd:nsd-catalog']
 621         elif clean_indata.get('nsd-catalog'):
 622             clean_indata = clean_indata['nsd-catalog']
 623         if clean_indata.get('nsd'):
 624             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 625                 raise EngineException("'nsd' must be a list of only one element")
 626             clean_indata = clean_indata['nsd'][0]
 627         elif clean_indata.get('nsd:nsd'):
 628             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 629                 raise EngineException("'nsd:nsd' must be a list of only one element")
 630             clean_indata = clean_indata['nsd:nsd'][0]
 631         return clean_indata
 632
 633     def _validate_input_new(self, indata, force=False):
 634         indata = self.pyangbind_validation("nsds", indata, force)
 635         # Cross references validation in the descriptor
 636         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 637         for vld in get_iterable(indata.get("vld")):
 638             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 639                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 640                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 641                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 642                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 643                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 644                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 645                                                                  constituent_vnfd["member-vnf-index"],
 646                                                                  constituent_vnfd["vnfd-id-ref"]),
 647                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 648                         break
 649                 else:
 650                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 651                                           "does not match any constituent-vnfd:member-vnf-index"
 652                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 653                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 654         return indata
 655
 656     def _validate_input_edit(self, indata, force=False):
 657         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 658         return indata
 659
 660     def _check_descriptor_dependencies(self, session, descriptor, force=False):
 661         """
 662         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 663         connection points are ok
 664         :param session: client session information
 665         :param descriptor: descriptor to be inserted or edit
 666         :param force: if true skip dependencies checking
 667         :return: None or raises exception
 668         """
 669         if force:
 670             return
 671         member_vnfd_index = {}
 672         if descriptor.get("constituent-vnfd") and not force:
 673             for vnf in descriptor["constituent-vnfd"]:
 674                 vnfd_id = vnf["vnfd-id-ref"]
 675                 filter_q = self._get_project_filter(session, write=False, show_all=True)
 676                 filter_q["id"] = vnfd_id
 677                 vnf_list = self.db.get_list("vnfds", filter_q)
 678                 if not vnf_list:
 679                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 680                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 681                 # elif len(vnf_list) > 1:
 682                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 683                 #                           http_code=HTTPStatus.CONFLICT)
 684                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 685
 686         # Cross references validation in the descriptor and vnfd connection point validation
 687         for vld in get_iterable(descriptor.get("vld")):
 688             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 689                 # look if this vnfd contains this connection point
 690                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 691                 if not vnfd:
 692                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 693                                           "does not match any constituent-vnfd:member-vnf-index"
 694                                           .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
 695                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 696                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 697                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 698                         break
 699                 else:
 700                     raise EngineException(
 701                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 702                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 703                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 704                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 705                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 706
 707     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 708         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 709
 710         self._check_descriptor_dependencies(session, final_content, force)
 711
 712     def check_conflict_on_del(self, session, _id, force=False):
 713         """
 714         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 715         that NSD can be public and be used by other projects.
 716         :param session:
 717         :param _id: vnfd inernal id
 718         :param force: Avoid this checking
 719         :return: None or raises EngineException with the conflict
 720         """
 721         if force:
 722             return
 723         _filter = self._get_project_filter(session, write=False, show_all=False)
 724         _filter["nsdId"] = _id
 725         if self.db.get_list("nsrs", _filter):
 726             raise EngineException("There is some NSR that depends on this NSD", http_code=HTTPStatus.CONFLICT)
 727
 728
 729 class NstTopic(DescriptorTopic):
 730     topic = "nsts"
 731     topic_msg = "nst"
 732
 733     def __init__(self, db, fs, msg):
 734         DescriptorTopic.__init__(self, db, fs, msg)
 735
 736     @staticmethod
 737     def _remove_envelop(indata=None):
 738         if not indata:
 739             return {}
 740         clean_indata = indata
 741
 742         if clean_indata.get('nst'):
 743             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 744                 raise EngineException("'nst' must be a list only one element")
 745             clean_indata = clean_indata['nst'][0]
 746         return clean_indata
 747
 748     def _validate_input_edit(self, indata, force=False):
 749         # TODO validate with pyangbind, serialize
 750         return indata
 751
 752     def _validate_input_new(self, indata, force=False):
 753         return indata.copy()
 754
 755     def _check_descriptor_dependencies(self, session, descriptor):
 756         """
 757         Check that the dependent descriptors exist on a new descriptor or edition
 758         :param session: client session information
 759         :param descriptor: descriptor to be inserted or edit
 760         :return: None or raises exception
 761         """
 762         if not descriptor.get("netslice-subnet"):
 763             return
 764         for nsd in descriptor["netslice-subnet"]:
 765             nsd_id = nsd["nsd-ref"]
 766             filter_q = self._get_project_filter(session, write=False, show_all=True)
 767             filter_q["id"] = nsd_id
 768             if not self.db.get_list("nsds", filter_q):
 769                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
 770                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
 771
 772     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 773         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 774
 775         self._check_descriptor_dependencies(session, final_content)
 776
 777     def check_conflict_on_del(self, session, _id, force=False):
 778         """
 779         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
 780         that NST can be public and be used by other projects.
 781         :param session:
 782         :param _id: nst internal id
 783         :param force: Avoid this checking
 784         :return: None or raises EngineException with the conflict
 785         """
 786         # TODO: Check this method
 787         if force:
 788             return
 789         # Get Network Slice Template from Database
 790         _filter = self._get_project_filter(session, write=False, show_all=False)
 791         _filter["_id"] = _id
 792         nst = self.db.get_one("nsts", _filter)
 793
 794         # Search NSIs using NST via nst-ref
 795         _filter = self._get_project_filter(session, write=False, show_all=False)
 796         _filter["nst-ref"] = nst["id"]
 797         nsis_list = self.db.get_list("nsis", _filter)
 798         for nsi_item in nsis_list:
 799             if nsi_item["_admin"].get("nsiState") != "TERMINATED":
 800                 raise EngineException("There is some NSIS that depends on this NST", http_code=HTTPStatus.CONFLICT)
 801
 802
 803 class PduTopic(BaseTopic):
 804     topic = "pdus"
 805     topic_msg = "pdu"
 806     schema_new = pdu_new_schema
 807     schema_edit = pdu_edit_schema
 808
 809     def __init__(self, db, fs, msg):
 810         BaseTopic.__init__(self, db, fs, msg)
 811
 812     @staticmethod
 813     def format_on_new(content, project_id=None, make_public=False):
 814         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 815         content["_admin"]["onboardingState"] = "CREATED"
 816         content["_admin"]["operationalState"] = "ENABLED"
 817         content["_admin"]["usageState"] = "NOT_IN_USE"
 818
 819     def check_conflict_on_del(self, session, _id, force=False):
 820         if force:
 821             return
 822         # TODO Is it needed to check descriptors _admin.project_read/project_write??
 823         _filter = {"vdur.pdu-id": _id}
 824         if self.db.get_list("vnfrs", _filter):
 825             raise EngineException("There is some NSR that uses this PDU", http_code=HTTPStatus.CONFLICT)