osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from validation import ValidationError, pdu_new_schema, pdu_edit_schema
  24 from base_topic import BaseTopic, EngineException, get_iterable
  25 from osm_im.vnfd import vnfd as vnfd_im
  26 from osm_im.nsd import nsd as nsd_im
  27 from osm_im.nst import nst as nst_im
  28 from pyangbind.lib.serialise import pybindJSONDecoder
  29 import pyangbind.lib.pybindJSON as pybindJSON
  30
  31 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  32
  33
  34 class DescriptorTopic(BaseTopic):
  35
  36     def __init__(self, db, fs, msg):
  37         BaseTopic.__init__(self, db, fs, msg)
  38
  39     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
  40         # 1. validate again with pyangbind
  41         # 1.1. remove internal keys
  42         internal_keys = {}
  43         for k in ("_id", "_admin"):
  44             if k in final_content:
  45                 internal_keys[k] = final_content.pop(k)
  46         serialized = self._validate_input_new(final_content, force)
  47         # 1.2. modify final_content with a serialized version
  48         final_content.clear()
  49         final_content.update(serialized)
  50         # 1.3. restore internal keys
  51         for k, v in internal_keys.items():
  52             final_content[k] = v
  53
  54         if force:
  55             return
  56         # 2. check that this id is not present
  57         if "id" in edit_content:
  58             _filter = self._get_project_filter(session, write=False, show_all=False)
  59             _filter["id"] = final_content["id"]
  60             _filter["_id.neq"] = _id
  61             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  62                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  63                                                                                                final_content["id"]),
  64                                       HTTPStatus.CONFLICT)
  65
  66     @staticmethod
  67     def format_on_new(content, project_id=None, make_public=False):
  68         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  69         content["_admin"]["onboardingState"] = "CREATED"
  70         content["_admin"]["operationalState"] = "DISABLED"
  71         content["_admin"]["usageState"] = "NOT_IN_USE"
  72
  73     def delete(self, session, _id, force=False, dry_run=False):
  74         """
  75         Delete item by its internal _id
  76         :param session: contains the used login username, working project, and admin rights
  77         :param _id: server internal id
  78         :param force: indicates if deletion must be forced in case of conflict
  79         :param dry_run: make checking but do not delete
  80         :return: dictionary with deleted item _id. It raises EngineException on error: not found, conflict, ...
  81         """
  82         # TODO add admin to filter, validate rights
  83         v = BaseTopic.delete(self, session, _id, force, dry_run=True)
  84         if dry_run:
  85             return
  86         v = self.db.del_one(self.topic, {"_id": _id})
  87         self.fs.file_delete(_id, ignore_non_exist=True)
  88         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
  89         self._send_msg("delete", {"_id": _id})
  90         return v
  91
  92     @staticmethod
  93     def get_one_by_id(db, session, topic, id):
  94         # find owned by this project
  95         _filter = BaseTopic._get_project_filter(session, write=False, show_all=False)
  96         _filter["id"] = id
  97         desc_list = db.get_list(topic, _filter)
  98         if len(desc_list) == 1:
  99             return desc_list[0]
 100         elif len(desc_list) > 1:
 101             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 102                               HTTPStatus.CONFLICT)
 103
 104         # not found any: try to find public
 105         _filter = BaseTopic._get_project_filter(session, write=False, show_all=True)
 106         _filter["id"] = id
 107         desc_list = db.get_list(topic, _filter)
 108         if not desc_list:
 109             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 110         elif len(desc_list) == 1:
 111             return desc_list[0]
 112         else:
 113             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 114                 topic[:-1], id), HTTPStatus.CONFLICT)
 115
 116     def new(self, rollback, session, indata=None, kwargs=None, headers=None, force=False, make_public=False):
 117         """
 118         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 119         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 120         (self.upload_content)
 121         :param rollback: list to append created items at database in case a rollback may to be done
 122         :param session: contains the used login username and working project
 123         :param indata: data to be inserted
 124         :param kwargs: used to override the indata descriptor
 125         :param headers: http request headers
 126         :param force: If True avoid some dependence checks
 127         :param make_public: Make the created descriptor public to all projects
 128         :return: _id: identity of the inserted data.
 129         """
 130
 131         try:
 132             # _remove_envelop
 133             if indata:
 134                 if "userDefinedData" in indata:
 135                     indata = indata['userDefinedData']
 136
 137             # Override descriptor with query string kwargs
 138             self._update_input_with_kwargs(indata, kwargs)
 139             # uncomment when this method is implemented.
 140             # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 141             # indata = DescriptorTopic._validate_input_new(self, indata, force=force)
 142
 143             content = {"_admin": {"userDefinedData": indata}}
 144             self.format_on_new(content, session["project_id"], make_public=make_public)
 145             _id = self.db.create(self.topic, content)
 146             rollback.append({"topic": self.topic, "_id": _id})
 147             return _id
 148         except ValidationError as e:
 149             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 150
 151     def upload_content(self, session, _id, indata, kwargs, headers, force=False):
 152         """
 153         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 154         :param session: session
 155         :param _id : the nsd,vnfd is already created, this is the id
 156         :param indata: http body request
 157         :param kwargs: user query string to override parameters. NOT USED
 158         :param headers:  http request headers
 159         :param force: to be more tolerant with validation
 160         :return: True if package is completely uploaded or False if partial content has been uploded
 161             Raise exception on error
 162         """
 163         # Check that _id exists and it is valid
 164         current_desc = self.show(session, _id)
 165
 166         content_range_text = headers.get("Content-Range")
 167         expected_md5 = headers.get("Content-File-MD5")
 168         compressed = None
 169         content_type = headers.get("Content-Type")
 170         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 171                 "application/zip" in content_type:
 172             compressed = "gzip"
 173         filename = headers.get("Content-Filename")
 174         if not filename:
 175             filename = "package.tar.gz" if compressed else "package"
 176         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 177         file_pkg = None
 178         error_text = ""
 179         try:
 180             if content_range_text:
 181                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 182                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 183                     raise IndexError()
 184                 start = int(content_range[1])
 185                 end = int(content_range[2]) + 1
 186                 total = int(content_range[3])
 187             else:
 188                 start = 0
 189             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 190
 191             if start:
 192                 if not self.fs.file_exists(temp_folder, 'dir'):
 193                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 194             else:
 195                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 196                 self.fs.mkdir(temp_folder)
 197
 198             storage = self.fs.get_params()
 199             storage["folder"] = _id
 200
 201             file_path = (temp_folder, filename)
 202             if self.fs.file_exists(file_path, 'file'):
 203                 file_size = self.fs.file_size(file_path)
 204             else:
 205                 file_size = 0
 206             if file_size != start:
 207                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 208                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 209             file_pkg = self.fs.file_open(file_path, 'a+b')
 210             if isinstance(indata, dict):
 211                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 212                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 213             else:
 214                 indata_len = 0
 215                 while True:
 216                     indata_text = indata.read(4096)
 217                     indata_len += len(indata_text)
 218                     if not indata_text:
 219                         break
 220                     file_pkg.write(indata_text)
 221             if content_range_text:
 222                 if indata_len != end-start:
 223                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 224                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 225                 if end != total:
 226                     # TODO update to UPLOADING
 227                     return False
 228
 229             # PACKAGE UPLOADED
 230             if expected_md5:
 231                 file_pkg.seek(0, 0)
 232                 file_md5 = md5()
 233                 chunk_data = file_pkg.read(1024)
 234                 while chunk_data:
 235                     file_md5.update(chunk_data)
 236                     chunk_data = file_pkg.read(1024)
 237                 if expected_md5 != file_md5.hexdigest():
 238                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 239             file_pkg.seek(0, 0)
 240             if compressed == "gzip":
 241                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 242                 descriptor_file_name = None
 243                 for tarinfo in tar:
 244                     tarname = tarinfo.name
 245                     tarname_path = tarname.split("/")
 246                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 247                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 248                     if len(tarname_path) == 1 and not tarinfo.isdir():
 249                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 250                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 251                         storage["pkg-dir"] = tarname_path[0]
 252                         if len(tarname_path) == 2:
 253                             if descriptor_file_name:
 254                                 raise EngineException(
 255                                     "Found more than one descriptor file at package descriptor tar.gz")
 256                             descriptor_file_name = tarname
 257                 if not descriptor_file_name:
 258                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 259                 storage["descriptor"] = descriptor_file_name
 260                 storage["zipfile"] = filename
 261                 self.fs.file_extract(tar, temp_folder)
 262                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 263                     content = descriptor_file.read()
 264             else:
 265                 content = file_pkg.read()
 266                 storage["descriptor"] = descriptor_file_name = filename
 267
 268             if descriptor_file_name.endswith(".json"):
 269                 error_text = "Invalid json format "
 270                 indata = json.load(content)
 271             else:
 272                 error_text = "Invalid yaml format "
 273                 indata = yaml.load(content)
 274
 275             current_desc["_admin"]["storage"] = storage
 276             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 277             current_desc["_admin"]["operationalState"] = "ENABLED"
 278
 279             indata = self._remove_envelop(indata)
 280
 281             # Override descriptor with query string kwargs
 282             if kwargs:
 283                 self._update_input_with_kwargs(indata, kwargs)
 284             # it will call overrides method at VnfdTopic or NsdTopic
 285             # indata = self._validate_input_edit(indata, force=force)
 286
 287             deep_update_rfc7396(current_desc, indata)
 288             self.check_conflict_on_edit(session, current_desc, indata, _id=_id, force=force)
 289             self.db.replace(self.topic, _id, current_desc)
 290             self.fs.dir_rename(temp_folder, _id)
 291
 292             indata["_id"] = _id
 293             self._send_msg("created", indata)
 294
 295             # TODO if descriptor has changed because kwargs update content and remove cached zip
 296             # TODO if zip is not present creates one
 297             return True
 298
 299         except EngineException:
 300             raise
 301         except IndexError:
 302             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 303                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 304         except IOError as e:
 305             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 306         except tarfile.ReadError as e:
 307             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 308         except (ValueError, yaml.YAMLError) as e:
 309             raise EngineException(error_text + str(e))
 310         except ValidationError as e:
 311             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 312         finally:
 313             if file_pkg:
 314                 file_pkg.close()
 315
 316     def get_file(self, session, _id, path=None, accept_header=None):
 317         """
 318         Return the file content of a vnfd or nsd
 319         :param session: contains the used login username and working project
 320         :param _id: Identity of the vnfd, nsd
 321         :param path: artifact path or "$DESCRIPTOR" or None
 322         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 323         :return: opened file plus Accept format or raises an exception
 324         """
 325         accept_text = accept_zip = False
 326         if accept_header:
 327             if 'text/plain' in accept_header or '*/*' in accept_header:
 328                 accept_text = True
 329             if 'application/zip' in accept_header or '*/*' in accept_header:
 330                 accept_zip = 'application/zip'
 331             elif 'application/gzip' in accept_header:
 332                 accept_zip = 'application/gzip'
 333
 334         if not accept_text and not accept_zip:
 335             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 336                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 337
 338         content = self.show(session, _id)
 339         if content["_admin"]["onboardingState"] != "ONBOARDED":
 340             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 341                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 342                                   http_code=HTTPStatus.CONFLICT)
 343         storage = content["_admin"]["storage"]
 344         if path is not None and path != "$DESCRIPTOR":   # artifacts
 345             if not storage.get('pkg-dir'):
 346                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 347             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 348                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 349                 return folder_content, "text/plain"
 350                 # TODO manage folders in http
 351             else:
 352                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 353                     "application/octet-stream"
 354
 355         # pkgtype   accept  ZIP  TEXT    -> result
 356         # manyfiles         yes  X       -> zip
 357         #                   no   yes     -> error
 358         # onefile           yes  no      -> zip
 359         #                   X    yes     -> text
 360
 361         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 362             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 363         elif storage.get('pkg-dir') and not accept_zip:
 364             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 365                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 366         else:
 367             if not storage.get('zipfile'):
 368                 # TODO generate zipfile if not present
 369                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 370                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 371             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 372
 373     def pyangbind_validation(self, item, data, force=False):
 374         try:
 375             if item == "vnfds":
 376                 myvnfd = vnfd_im()
 377                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 378                                                  path_helper=True, skip_unknown=force)
 379                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 380             elif item == "nsds":
 381                 mynsd = nsd_im()
 382                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 383                                                  path_helper=True, skip_unknown=force)
 384                 out = pybindJSON.dumps(mynsd, mode="ietf")
 385             elif item == "nsts":
 386                 mynst = nst_im()
 387                 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
 388                                                  path_helper=True, skip_unknown=force)
 389                 out = pybindJSON.dumps(mynst, mode="ietf")
 390             else:
 391                 raise EngineException("Not possible to validate '{}' item".format(item),
 392                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 393
 394             desc_out = self._remove_envelop(yaml.safe_load(out))
 395             return desc_out
 396
 397         except Exception as e:
 398             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 399                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 400
 401
 402 class VnfdTopic(DescriptorTopic):
 403     topic = "vnfds"
 404     topic_msg = "vnfd"
 405
 406     def __init__(self, db, fs, msg):
 407         DescriptorTopic.__init__(self, db, fs, msg)
 408
 409     @staticmethod
 410     def _remove_envelop(indata=None):
 411         if not indata:
 412             return {}
 413         clean_indata = indata
 414         if clean_indata.get('vnfd:vnfd-catalog'):
 415             clean_indata = clean_indata['vnfd:vnfd-catalog']
 416         elif clean_indata.get('vnfd-catalog'):
 417             clean_indata = clean_indata['vnfd-catalog']
 418         if clean_indata.get('vnfd'):
 419             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 420                 raise EngineException("'vnfd' must be a list of only one element")
 421             clean_indata = clean_indata['vnfd'][0]
 422         elif clean_indata.get('vnfd:vnfd'):
 423             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 424                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 425             clean_indata = clean_indata['vnfd:vnfd'][0]
 426         return clean_indata
 427
 428     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 429         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 430
 431         # set type of vnfd
 432         contains_pdu = False
 433         contains_vdu = False
 434         for vdu in get_iterable(final_content.get("vdu")):
 435             if vdu.get("pdu-type"):
 436                 contains_pdu = True
 437             else:
 438                 contains_vdu = True
 439         if contains_pdu:
 440             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 441         elif contains_vdu:
 442             final_content["_admin"]["type"] = "vnfd"
 443         # if neither vud nor pdu do not fill type
 444
 445     def check_conflict_on_del(self, session, _id, force=False):
 446         """
 447         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 448         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 449         that uses this vnfd
 450         :param session:
 451         :param _id: vnfd inernal id
 452         :param force: Avoid this checking
 453         :return: None or raises EngineException with the conflict
 454         """
 455         if force:
 456             return
 457         descriptor = self.db.get_one("vnfds", {"_id": _id})
 458         descriptor_id = descriptor.get("id")
 459         if not descriptor_id:  # empty vnfd not uploaded
 460             return
 461
 462         _filter = self._get_project_filter(session, write=False, show_all=False)
 463         # check vnfrs using this vnfd
 464         _filter["vnfd-id"] = _id
 465         if self.db.get_list("vnfrs", _filter):
 466             raise EngineException("There is some VNFR that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 467         del _filter["vnfd-id"]
 468         # check NSD using this VNFD
 469         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 470         if self.db.get_list("nsds", _filter):
 471             raise EngineException("There is soame NSD that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 472
 473     def _validate_input_new(self, indata, force=False):
 474         indata = self.pyangbind_validation("vnfds", indata, force)
 475         # Cross references validation in the descriptor
 476         if indata.get("vdu"):
 477             if not indata.get("mgmt-interface"):
 478                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 479                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 480             if indata["mgmt-interface"].get("cp"):
 481                 for cp in get_iterable(indata.get("connection-point")):
 482                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 483                         break
 484                 else:
 485                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 486                                           .format(indata["mgmt-interface"]["cp"]),
 487                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 488
 489         for vdu in get_iterable(indata.get("vdu")):
 490             for interface in get_iterable(vdu.get("interface")):
 491                 if interface.get("external-connection-point-ref"):
 492                     for cp in get_iterable(indata.get("connection-point")):
 493                         if cp["name"] == interface["external-connection-point-ref"]:
 494                             break
 495                     else:
 496                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 497                                               "must match an existing connection-point"
 498                                               .format(vdu["id"], interface["name"],
 499                                                       interface["external-connection-point-ref"]),
 500                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 501
 502                 elif interface.get("internal-connection-point-ref"):
 503                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 504                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 505                             break
 506                     else:
 507                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 508                                               "must match an existing vdu:internal-connection-point"
 509                                               .format(vdu["id"], interface["name"],
 510                                                       interface["internal-connection-point-ref"]),
 511                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 512         for ivld in get_iterable(indata.get("internal-vld")):
 513             for icp in get_iterable(ivld.get("internal-connection-point")):
 514                 icp_mark = False
 515                 for vdu in get_iterable(indata.get("vdu")):
 516                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 517                         if icp["id-ref"] == internal_cp["id"]:
 518                             icp_mark = True
 519                             break
 520                     if icp_mark:
 521                         break
 522                 else:
 523                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 524                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 525                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 526             if ivld.get("ip-profile-ref"):
 527                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 528                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 529                         break
 530                 else:
 531                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 532                         ivld["id"], ivld["ip-profile-ref"]),
 533                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 534         for mp in get_iterable(indata.get("monitoring-param")):
 535             if mp.get("vdu-monitoring-param"):
 536                 mp_vmp_mark = False
 537                 for vdu in get_iterable(indata.get("vdu")):
 538                     for vmp in get_iterable(vdu.get("monitoring-param")):
 539                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 540                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 541                             mp_vmp_mark = True
 542                             break
 543                     if mp_vmp_mark:
 544                         break
 545                 else:
 546                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 547                                           "defined at vdu[id='{}'] or vdu does not exist"
 548                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 549                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 550                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 551             elif mp.get("vdu-metric"):
 552                 mp_vm_mark = False
 553                 for vdu in get_iterable(indata.get("vdu")):
 554                     if vdu.get("vdu-configuration"):
 555                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 556                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 557                                     mp["vdu-metric"]["vdu-ref"]:
 558                                 mp_vm_mark = True
 559                                 break
 560                         if mp_vm_mark:
 561                             break
 562                 else:
 563                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 564                                           "vdu[id='{}'] or vdu does not exist"
 565                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 566                                                   mp["vdu-metric"]["vdu-ref"]),
 567                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 568
 569         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 570             for sp in get_iterable(sgd.get("scaling-policy")):
 571                 for sc in get_iterable(sp.get("scaling-criteria")):
 572                     for mp in get_iterable(indata.get("monitoring-param")):
 573                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 574                             break
 575                     else:
 576                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 577                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 578                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 579                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 580             for sgd_vdu in get_iterable(sgd.get("vdu")):
 581                 sgd_vdu_mark = False
 582                 for vdu in get_iterable(indata.get("vdu")):
 583                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 584                         sgd_vdu_mark = True
 585                         break
 586                 if sgd_vdu_mark:
 587                     break
 588             else:
 589                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 590                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 591                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 592             for sca in get_iterable(sgd.get("scaling-config-action")):
 593                 if not indata.get("vnf-configuration"):
 594                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 595                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 596                                           .format(sgd["name"]),
 597                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 598                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 599                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 600                         break
 601                 else:
 602                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 603                                           "primitive-name-ref='{}' does not match any "
 604                                           "vnf-configuration:config-primitive:name"
 605                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 606                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 607         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 608         return indata
 609
 610     def _validate_input_edit(self, indata, force=False):
 611         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 612         return indata
 613
 614
 615 class NsdTopic(DescriptorTopic):
 616     topic = "nsds"
 617     topic_msg = "nsd"
 618
 619     def __init__(self, db, fs, msg):
 620         DescriptorTopic.__init__(self, db, fs, msg)
 621
 622     @staticmethod
 623     def _remove_envelop(indata=None):
 624         if not indata:
 625             return {}
 626         clean_indata = indata
 627
 628         if clean_indata.get('nsd:nsd-catalog'):
 629             clean_indata = clean_indata['nsd:nsd-catalog']
 630         elif clean_indata.get('nsd-catalog'):
 631             clean_indata = clean_indata['nsd-catalog']
 632         if clean_indata.get('nsd'):
 633             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 634                 raise EngineException("'nsd' must be a list of only one element")
 635             clean_indata = clean_indata['nsd'][0]
 636         elif clean_indata.get('nsd:nsd'):
 637             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 638                 raise EngineException("'nsd:nsd' must be a list of only one element")
 639             clean_indata = clean_indata['nsd:nsd'][0]
 640         return clean_indata
 641
 642     def _validate_input_new(self, indata, force=False):
 643         indata = self.pyangbind_validation("nsds", indata, force)
 644         # Cross references validation in the descriptor
 645         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 646         for vld in get_iterable(indata.get("vld")):
 647             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 648                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 649                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 650                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 651                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 652                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 653                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 654                                                                  constituent_vnfd["member-vnf-index"],
 655                                                                  constituent_vnfd["vnfd-id-ref"]),
 656                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 657                         break
 658                 else:
 659                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 660                                           "does not match any constituent-vnfd:member-vnf-index"
 661                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 662                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 663         return indata
 664
 665     def _validate_input_edit(self, indata, force=False):
 666         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 667         return indata
 668
 669     def _check_descriptor_dependencies(self, session, descriptor, force=False):
 670         """
 671         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 672         connection points are ok
 673         :param session: client session information
 674         :param descriptor: descriptor to be inserted or edit
 675         :param force: if true skip dependencies checking
 676         :return: None or raises exception
 677         """
 678         if force:
 679             return
 680         member_vnfd_index = {}
 681         if descriptor.get("constituent-vnfd") and not force:
 682             for vnf in descriptor["constituent-vnfd"]:
 683                 vnfd_id = vnf["vnfd-id-ref"]
 684                 filter_q = self._get_project_filter(session, write=False, show_all=True)
 685                 filter_q["id"] = vnfd_id
 686                 vnf_list = self.db.get_list("vnfds", filter_q)
 687                 if not vnf_list:
 688                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 689                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 690                 # elif len(vnf_list) > 1:
 691                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 692                 #                           http_code=HTTPStatus.CONFLICT)
 693                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 694
 695         # Cross references validation in the descriptor and vnfd connection point validation
 696         for vld in get_iterable(descriptor.get("vld")):
 697             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 698                 # look if this vnfd contains this connection point
 699                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 700                 if not vnfd:
 701                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 702                                           "does not match any constituent-vnfd:member-vnf-index"
 703                                           .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
 704                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 705                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 706                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 707                         break
 708                 else:
 709                     raise EngineException(
 710                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 711                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 712                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 713                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 714                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 715
 716     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 717         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 718
 719         self._check_descriptor_dependencies(session, final_content, force)
 720
 721     def check_conflict_on_del(self, session, _id, force=False):
 722         """
 723         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 724         that NSD can be public and be used by other projects.
 725         :param session:
 726         :param _id: vnfd inernal id
 727         :param force: Avoid this checking
 728         :return: None or raises EngineException with the conflict
 729         """
 730         if force:
 731             return
 732         _filter = self._get_project_filter(session, write=False, show_all=False)
 733         _filter["nsdId"] = _id
 734         if self.db.get_list("nsrs", _filter):
 735             raise EngineException("There is some NSR that depends on this NSD", http_code=HTTPStatus.CONFLICT)
 736
 737
 738 class NstTopic(DescriptorTopic):
 739     topic = "nsts"
 740     topic_msg = "nst"
 741
 742     def __init__(self, db, fs, msg):
 743         DescriptorTopic.__init__(self, db, fs, msg)
 744
 745     @staticmethod
 746     def _remove_envelop(indata=None):
 747         if not indata:
 748             return {}
 749         clean_indata = indata
 750
 751         if clean_indata.get('nst'):
 752             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 753                 raise EngineException("'nst' must be a list only one element")
 754             clean_indata = clean_indata['nst'][0]
 755         elif clean_indata.get('nst:nst'):
 756             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
 757                 raise EngineException("'nst:nst' must be a list only one element")
 758             clean_indata = clean_indata['nst:nst'][0]
 759         return clean_indata
 760
 761     def _validate_input_edit(self, indata, force=False):
 762         # TODO validate with pyangbind, serialize
 763         return indata
 764
 765     def _validate_input_new(self, indata, force=False):
 766         indata = self.pyangbind_validation("nsts", indata, force)
 767         return indata.copy()
 768
 769     def _check_descriptor_dependencies(self, session, descriptor):
 770         """
 771         Check that the dependent descriptors exist on a new descriptor or edition
 772         :param session: client session information
 773         :param descriptor: descriptor to be inserted or edit
 774         :return: None or raises exception
 775         """
 776         if not descriptor.get("netslice-subnet"):
 777             return
 778         for nsd in descriptor["netslice-subnet"]:
 779             nsd_id = nsd["nsd-ref"]
 780             filter_q = self._get_project_filter(session, write=False, show_all=True)
 781             filter_q["id"] = nsd_id
 782             if not self.db.get_list("nsds", filter_q):
 783                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
 784                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
 785
 786     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
 787         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 788
 789         self._check_descriptor_dependencies(session, final_content)
 790
 791     def check_conflict_on_del(self, session, _id, force=False):
 792         """
 793         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
 794         that NST can be public and be used by other projects.
 795         :param session:
 796         :param _id: nst internal id
 797         :param force: Avoid this checking
 798         :return: None or raises EngineException with the conflict
 799         """
 800         # TODO: Check this method
 801         if force:
 802             return
 803         # Get Network Slice Template from Database
 804         _filter = self._get_project_filter(session, write=False, show_all=False)
 805         _filter["_id"] = _id
 806         nst = self.db.get_one("nsts", _filter)
 807
 808         # Search NSIs using NST via nst-ref
 809         _filter = self._get_project_filter(session, write=False, show_all=False)
 810         _filter["nst-ref"] = nst["id"]
 811         nsis_list = self.db.get_list("nsis", _filter)
 812         for nsi_item in nsis_list:
 813             if nsi_item["_admin"].get("nsiState") != "TERMINATED":
 814                 raise EngineException("There is some NSIS that depends on this NST", http_code=HTTPStatus.CONFLICT)
 815
 816
 817 class PduTopic(BaseTopic):
 818     topic = "pdus"
 819     topic_msg = "pdu"
 820     schema_new = pdu_new_schema
 821     schema_edit = pdu_edit_schema
 822
 823     def __init__(self, db, fs, msg):
 824         BaseTopic.__init__(self, db, fs, msg)
 825
 826     @staticmethod
 827     def format_on_new(content, project_id=None, make_public=False):
 828         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 829         content["_admin"]["onboardingState"] = "CREATED"
 830         content["_admin"]["operationalState"] = "ENABLED"
 831         content["_admin"]["usageState"] = "NOT_IN_USE"
 832
 833     def check_conflict_on_del(self, session, _id, force=False):
 834         if force:
 835             return
 836         # TODO Is it needed to check descriptors _admin.project_read/project_write??
 837         _filter = {"vdur.pdu-id": _id}
 838         if self.db.get_list("vnfrs", _filter):
 839             raise EngineException("There is some NSR that uses this PDU", http_code=HTTPStatus.CONFLICT)