osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from time import time
  24 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema
  25 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  26 from osm_im.vnfd import vnfd as vnfd_im
  27 from osm_im.nsd import nsd as nsd_im
  28 from osm_im.nst import nst as nst_im
  29 from pyangbind.lib.serialise import pybindJSONDecoder
  30 import pyangbind.lib.pybindJSON as pybindJSON
  31
  32 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  33
  34
  35 class DescriptorTopic(BaseTopic):
  36
  37     def __init__(self, db, fs, msg, auth):
  38         BaseTopic.__init__(self, db, fs, msg, auth)
  39
  40     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  41         super().check_conflict_on_edit(session, final_content, edit_content, _id)
  42
  43         def _check_unique_id_name(descriptor, position=""):
  44             for desc_key, desc_item in descriptor.items():
  45                 if isinstance(desc_item, list) and desc_item:
  46                     used_ids = []
  47                     desc_item_id = None
  48                     for index, list_item in enumerate(desc_item):
  49                         if isinstance(list_item, dict):
  50                             _check_unique_id_name(list_item, "{}.{}[{}]"
  51                                                   .format(position, desc_key, index))
  52                             # Base case
  53                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  54                                 desc_item_id = "id" if list_item.get("id") else "name"
  55                             if desc_item_id and list_item.get(desc_item_id):
  56                                 if list_item[desc_item_id] in used_ids:
  57                                     position = "{}.{}[{}]".format(position, desc_key, index)
  58                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  59                                                           .format(desc_item_id, list_item[desc_item_id],
  60                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  61                                 used_ids.append(list_item[desc_item_id])
  62         _check_unique_id_name(final_content)
  63         # 1. validate again with pyangbind
  64         # 1.1. remove internal keys
  65         internal_keys = {}
  66         for k in ("_id", "_admin"):
  67             if k in final_content:
  68                 internal_keys[k] = final_content.pop(k)
  69         storage_params = internal_keys["_admin"].get("storage")
  70         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  71         # 1.2. modify final_content with a serialized version
  72         final_content.clear()
  73         final_content.update(serialized)
  74         # 1.3. restore internal keys
  75         for k, v in internal_keys.items():
  76             final_content[k] = v
  77
  78         if session["force"]:
  79             return
  80         # 2. check that this id is not present
  81         if "id" in edit_content:
  82             _filter = self._get_project_filter(session)
  83             _filter["id"] = final_content["id"]
  84             _filter["_id.neq"] = _id
  85             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  86                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  87                                                                                                final_content["id"]),
  88                                       HTTPStatus.CONFLICT)
  89
  90     @staticmethod
  91     def format_on_new(content, project_id=None, make_public=False):
  92         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  93         content["_admin"]["onboardingState"] = "CREATED"
  94         content["_admin"]["operationalState"] = "DISABLED"
  95         content["_admin"]["usageState"] = "NOT_IN_USE"
  96
  97     def delete_extra(self, session, _id, db_content, not_send_msg=None):
  98         """
  99         Deletes file system storage associated with the descriptor
 100         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 101         :param _id: server internal id
 102         :param db_content: The database content of the descriptor
 103         :param not_send_msg: To not send message (False) or store content (list) instead
 104         :return: None if ok or raises EngineException with the problem
 105         """
 106         self.fs.file_delete(_id, ignore_non_exist=True)
 107         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 108
 109     @staticmethod
 110     def get_one_by_id(db, session, topic, id):
 111         # find owned by this project
 112         _filter = BaseTopic._get_project_filter(session)
 113         _filter["id"] = id
 114         desc_list = db.get_list(topic, _filter)
 115         if len(desc_list) == 1:
 116             return desc_list[0]
 117         elif len(desc_list) > 1:
 118             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 119                               HTTPStatus.CONFLICT)
 120
 121         # not found any: try to find public
 122         _filter = BaseTopic._get_project_filter(session)
 123         _filter["id"] = id
 124         desc_list = db.get_list(topic, _filter)
 125         if not desc_list:
 126             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 127         elif len(desc_list) == 1:
 128             return desc_list[0]
 129         else:
 130             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 131                 topic[:-1], id), HTTPStatus.CONFLICT)
 132
 133     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 134         """
 135         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 136         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 137         (self.upload_content)
 138         :param rollback: list to append created items at database in case a rollback may to be done
 139         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 140         :param indata: data to be inserted
 141         :param kwargs: used to override the indata descriptor
 142         :param headers: http request headers
 143         :return: _id, None: identity of the inserted data; and None as there is not any operation
 144         """
 145
 146         try:
 147             # Check Quota
 148             self.check_quota(session)
 149
 150             # _remove_envelop
 151             if indata:
 152                 if "userDefinedData" in indata:
 153                     indata = indata['userDefinedData']
 154
 155             # Override descriptor with query string kwargs
 156             self._update_input_with_kwargs(indata, kwargs)
 157             # uncomment when this method is implemented.
 158             # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 159             # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 160
 161             content = {"_admin": {"userDefinedData": indata}}
 162             self.format_on_new(content, session["project_id"], make_public=session["public"])
 163             _id = self.db.create(self.topic, content)
 164             rollback.append({"topic": self.topic, "_id": _id})
 165             self._send_msg("created", {"_id": _id})
 166             return _id, None
 167         except ValidationError as e:
 168             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 169
 170     def upload_content(self, session, _id, indata, kwargs, headers):
 171         """
 172         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 173         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 174         :param _id : the nsd,vnfd is already created, this is the id
 175         :param indata: http body request
 176         :param kwargs: user query string to override parameters. NOT USED
 177         :param headers:  http request headers
 178         :return: True if package is completely uploaded or False if partial content has been uploded
 179             Raise exception on error
 180         """
 181         # Check that _id exists and it is valid
 182         current_desc = self.show(session, _id)
 183
 184         content_range_text = headers.get("Content-Range")
 185         expected_md5 = headers.get("Content-File-MD5")
 186         compressed = None
 187         content_type = headers.get("Content-Type")
 188         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 189                 "application/zip" in content_type:
 190             compressed = "gzip"
 191         filename = headers.get("Content-Filename")
 192         if not filename:
 193             filename = "package.tar.gz" if compressed else "package"
 194         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 195         file_pkg = None
 196         error_text = ""
 197         try:
 198             if content_range_text:
 199                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 200                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 201                     raise IndexError()
 202                 start = int(content_range[1])
 203                 end = int(content_range[2]) + 1
 204                 total = int(content_range[3])
 205             else:
 206                 start = 0
 207             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 208
 209             if start:
 210                 if not self.fs.file_exists(temp_folder, 'dir'):
 211                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 212             else:
 213                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 214                 self.fs.mkdir(temp_folder)
 215
 216             storage = self.fs.get_params()
 217             storage["folder"] = _id
 218
 219             file_path = (temp_folder, filename)
 220             if self.fs.file_exists(file_path, 'file'):
 221                 file_size = self.fs.file_size(file_path)
 222             else:
 223                 file_size = 0
 224             if file_size != start:
 225                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 226                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 227             file_pkg = self.fs.file_open(file_path, 'a+b')
 228             if isinstance(indata, dict):
 229                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 230                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 231             else:
 232                 indata_len = 0
 233                 while True:
 234                     indata_text = indata.read(4096)
 235                     indata_len += len(indata_text)
 236                     if not indata_text:
 237                         break
 238                     file_pkg.write(indata_text)
 239             if content_range_text:
 240                 if indata_len != end-start:
 241                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 242                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 243                 if end != total:
 244                     # TODO update to UPLOADING
 245                     return False
 246
 247             # PACKAGE UPLOADED
 248             if expected_md5:
 249                 file_pkg.seek(0, 0)
 250                 file_md5 = md5()
 251                 chunk_data = file_pkg.read(1024)
 252                 while chunk_data:
 253                     file_md5.update(chunk_data)
 254                     chunk_data = file_pkg.read(1024)
 255                 if expected_md5 != file_md5.hexdigest():
 256                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 257             file_pkg.seek(0, 0)
 258             if compressed == "gzip":
 259                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 260                 descriptor_file_name = None
 261                 for tarinfo in tar:
 262                     tarname = tarinfo.name
 263                     tarname_path = tarname.split("/")
 264                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 265                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 266                     if len(tarname_path) == 1 and not tarinfo.isdir():
 267                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 268                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 269                         storage["pkg-dir"] = tarname_path[0]
 270                         if len(tarname_path) == 2:
 271                             if descriptor_file_name:
 272                                 raise EngineException(
 273                                     "Found more than one descriptor file at package descriptor tar.gz")
 274                             descriptor_file_name = tarname
 275                 if not descriptor_file_name:
 276                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 277                 storage["descriptor"] = descriptor_file_name
 278                 storage["zipfile"] = filename
 279                 self.fs.file_extract(tar, temp_folder)
 280                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 281                     content = descriptor_file.read()
 282             else:
 283                 content = file_pkg.read()
 284                 storage["descriptor"] = descriptor_file_name = filename
 285
 286             if descriptor_file_name.endswith(".json"):
 287                 error_text = "Invalid json format "
 288                 indata = json.load(content)
 289             else:
 290                 error_text = "Invalid yaml format "
 291                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 292
 293             current_desc["_admin"]["storage"] = storage
 294             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 295             current_desc["_admin"]["operationalState"] = "ENABLED"
 296
 297             indata = self._remove_envelop(indata)
 298
 299             # Override descriptor with query string kwargs
 300             if kwargs:
 301                 self._update_input_with_kwargs(indata, kwargs)
 302             # it will call overrides method at VnfdTopic or NsdTopic
 303             # indata = self._validate_input_edit(indata, force=session["force"])
 304
 305             deep_update_rfc7396(current_desc, indata)
 306             self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 307             current_desc["_admin"]["modified"] = time()
 308             self.db.replace(self.topic, _id, current_desc)
 309             self.fs.dir_rename(temp_folder, _id)
 310
 311             indata["_id"] = _id
 312             self._send_msg("edited", indata)
 313
 314             # TODO if descriptor has changed because kwargs update content and remove cached zip
 315             # TODO if zip is not present creates one
 316             return True
 317
 318         except EngineException:
 319             raise
 320         except IndexError:
 321             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 322                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 323         except IOError as e:
 324             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 325         except tarfile.ReadError as e:
 326             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 327         except (ValueError, yaml.YAMLError) as e:
 328             raise EngineException(error_text + str(e))
 329         except ValidationError as e:
 330             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 331         finally:
 332             if file_pkg:
 333                 file_pkg.close()
 334
 335     def get_file(self, session, _id, path=None, accept_header=None):
 336         """
 337         Return the file content of a vnfd or nsd
 338         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 339         :param _id: Identity of the vnfd, nsd
 340         :param path: artifact path or "$DESCRIPTOR" or None
 341         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 342         :return: opened file plus Accept format or raises an exception
 343         """
 344         accept_text = accept_zip = False
 345         if accept_header:
 346             if 'text/plain' in accept_header or '*/*' in accept_header:
 347                 accept_text = True
 348             if 'application/zip' in accept_header or '*/*' in accept_header:
 349                 accept_zip = 'application/zip'
 350             elif 'application/gzip' in accept_header:
 351                 accept_zip = 'application/gzip'
 352
 353         if not accept_text and not accept_zip:
 354             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 355                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 356
 357         content = self.show(session, _id)
 358         if content["_admin"]["onboardingState"] != "ONBOARDED":
 359             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 360                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 361                                   http_code=HTTPStatus.CONFLICT)
 362         storage = content["_admin"]["storage"]
 363         if path is not None and path != "$DESCRIPTOR":   # artifacts
 364             if not storage.get('pkg-dir'):
 365                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 366             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 367                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 368                 return folder_content, "text/plain"
 369                 # TODO manage folders in http
 370             else:
 371                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 372                     "application/octet-stream"
 373
 374         # pkgtype   accept  ZIP  TEXT    -> result
 375         # manyfiles         yes  X       -> zip
 376         #                   no   yes     -> error
 377         # onefile           yes  no      -> zip
 378         #                   X    yes     -> text
 379
 380         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 381             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 382         elif storage.get('pkg-dir') and not accept_zip:
 383             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 384                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 385         else:
 386             if not storage.get('zipfile'):
 387                 # TODO generate zipfile if not present
 388                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 389                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 390             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 391
 392     def pyangbind_validation(self, item, data, force=False):
 393         try:
 394             if item == "vnfds":
 395                 myvnfd = vnfd_im()
 396                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 397                                                  path_helper=True, skip_unknown=force)
 398                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 399             elif item == "nsds":
 400                 mynsd = nsd_im()
 401                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 402                                                  path_helper=True, skip_unknown=force)
 403                 out = pybindJSON.dumps(mynsd, mode="ietf")
 404             elif item == "nsts":
 405                 mynst = nst_im()
 406                 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
 407                                                  path_helper=True, skip_unknown=force)
 408                 out = pybindJSON.dumps(mynst, mode="ietf")
 409             else:
 410                 raise EngineException("Not possible to validate '{}' item".format(item),
 411                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 412
 413             desc_out = self._remove_envelop(yaml.safe_load(out))
 414             return desc_out
 415
 416         except Exception as e:
 417             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 418                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 419
 420
 421 class VnfdTopic(DescriptorTopic):
 422     topic = "vnfds"
 423     topic_msg = "vnfd"
 424
 425     def __init__(self, db, fs, msg, auth):
 426         DescriptorTopic.__init__(self, db, fs, msg, auth)
 427
 428     @staticmethod
 429     def _remove_envelop(indata=None):
 430         if not indata:
 431             return {}
 432         clean_indata = indata
 433         if clean_indata.get('vnfd:vnfd-catalog'):
 434             clean_indata = clean_indata['vnfd:vnfd-catalog']
 435         elif clean_indata.get('vnfd-catalog'):
 436             clean_indata = clean_indata['vnfd-catalog']
 437         if clean_indata.get('vnfd'):
 438             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 439                 raise EngineException("'vnfd' must be a list of only one element")
 440             clean_indata = clean_indata['vnfd'][0]
 441         elif clean_indata.get('vnfd:vnfd'):
 442             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 443                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 444             clean_indata = clean_indata['vnfd:vnfd'][0]
 445         return clean_indata
 446
 447     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 448         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 449
 450         # set type of vnfd
 451         contains_pdu = False
 452         contains_vdu = False
 453         for vdu in get_iterable(final_content.get("vdu")):
 454             if vdu.get("pdu-type"):
 455                 contains_pdu = True
 456             else:
 457                 contains_vdu = True
 458         if contains_pdu:
 459             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 460         elif contains_vdu:
 461             final_content["_admin"]["type"] = "vnfd"
 462         # if neither vud nor pdu do not fill type
 463
 464     def check_conflict_on_del(self, session, _id, db_content):
 465         """
 466         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 467         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 468         that uses this vnfd
 469         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 470         :param _id: vnfd internal id
 471         :param db_content: The database content of the _id.
 472         :return: None or raises EngineException with the conflict
 473         """
 474         if session["force"]:
 475             return
 476         descriptor = db_content
 477         descriptor_id = descriptor.get("id")
 478         if not descriptor_id:  # empty vnfd not uploaded
 479             return
 480
 481         _filter = self._get_project_filter(session)
 482
 483         # check vnfrs using this vnfd
 484         _filter["vnfd-id"] = _id
 485         if self.db.get_list("vnfrs", _filter):
 486             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 487
 488         # check NSD referencing this VNFD
 489         del _filter["vnfd-id"]
 490         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 491         if self.db.get_list("nsds", _filter):
 492             raise EngineException("There is at least one NSD referencing this descriptor",
 493                                   http_code=HTTPStatus.CONFLICT)
 494
 495     def _validate_input_new(self, indata, storage_params, force=False):
 496         indata = self.pyangbind_validation("vnfds", indata, force)
 497         # Cross references validation in the descriptor
 498         if indata.get("vdu"):
 499             if not indata.get("mgmt-interface"):
 500                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 501                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 502             if indata["mgmt-interface"].get("cp"):
 503                 for cp in get_iterable(indata.get("connection-point")):
 504                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 505                         break
 506                 else:
 507                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 508                                           .format(indata["mgmt-interface"]["cp"]),
 509                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 510
 511         for vdu in get_iterable(indata.get("vdu")):
 512             for interface in get_iterable(vdu.get("interface")):
 513                 if interface.get("external-connection-point-ref"):
 514                     for cp in get_iterable(indata.get("connection-point")):
 515                         if cp["name"] == interface["external-connection-point-ref"]:
 516                             break
 517                     else:
 518                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 519                                               "must match an existing connection-point"
 520                                               .format(vdu["id"], interface["name"],
 521                                                       interface["external-connection-point-ref"]),
 522                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 523
 524                 elif interface.get("internal-connection-point-ref"):
 525                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 526                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 527                             break
 528                     else:
 529                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 530                                               "must match an existing vdu:internal-connection-point"
 531                                               .format(vdu["id"], interface["name"],
 532                                                       interface["internal-connection-point-ref"]),
 533                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 534             # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 535             if vdu.get("vdu-configuration"):
 536                 if vdu["vdu-configuration"].get("juju"):
 537                     if not self._validate_package_folders(storage_params, 'charms'):
 538                         raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
 539                                               "package".format(indata["id"], vdu["id"]))
 540             # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
 541             if vdu.get("cloud-init-file"):
 542                 if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 543                     raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 544                                           "package".format(indata["id"], vdu["id"]))
 545         # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 546         if indata.get("vnf-configuration"):
 547             if indata["vnf-configuration"].get("juju"):
 548                 if not self._validate_package_folders(storage_params, 'charms'):
 549                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 550                                           "package".format(indata["id"]))
 551         vld_names = []  # For detection of duplicated VLD names
 552         for ivld in get_iterable(indata.get("internal-vld")):
 553             # BEGIN Detection of duplicated VLD names
 554             ivld_name = ivld["name"]
 555             if ivld_name in vld_names:
 556                 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
 557                                       .format(ivld["name"], indata["id"], ivld["id"]),
 558                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 559             else:
 560                 vld_names.append(ivld_name)
 561             # END Detection of duplicated VLD names
 562             for icp in get_iterable(ivld.get("internal-connection-point")):
 563                 icp_mark = False
 564                 for vdu in get_iterable(indata.get("vdu")):
 565                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 566                         if icp["id-ref"] == internal_cp["id"]:
 567                             icp_mark = True
 568                             break
 569                     if icp_mark:
 570                         break
 571                 else:
 572                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 573                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 574                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 575             if ivld.get("ip-profile-ref"):
 576                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 577                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 578                         break
 579                 else:
 580                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 581                         ivld["id"], ivld["ip-profile-ref"]),
 582                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 583         for mp in get_iterable(indata.get("monitoring-param")):
 584             if mp.get("vdu-monitoring-param"):
 585                 mp_vmp_mark = False
 586                 for vdu in get_iterable(indata.get("vdu")):
 587                     for vmp in get_iterable(vdu.get("monitoring-param")):
 588                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 589                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 590                             mp_vmp_mark = True
 591                             break
 592                     if mp_vmp_mark:
 593                         break
 594                 else:
 595                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 596                                           "defined at vdu[id='{}'] or vdu does not exist"
 597                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 598                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 599                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 600             elif mp.get("vdu-metric"):
 601                 mp_vm_mark = False
 602                 for vdu in get_iterable(indata.get("vdu")):
 603                     if vdu.get("vdu-configuration"):
 604                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 605                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 606                                     mp["vdu-metric"]["vdu-ref"]:
 607                                 mp_vm_mark = True
 608                                 break
 609                         if mp_vm_mark:
 610                             break
 611                 else:
 612                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 613                                           "vdu[id='{}'] or vdu does not exist"
 614                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 615                                                   mp["vdu-metric"]["vdu-ref"]),
 616                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 617
 618         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 619             for sp in get_iterable(sgd.get("scaling-policy")):
 620                 for sc in get_iterable(sp.get("scaling-criteria")):
 621                     for mp in get_iterable(indata.get("monitoring-param")):
 622                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 623                             break
 624                     else:
 625                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 626                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 627                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 628                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 629             for sgd_vdu in get_iterable(sgd.get("vdu")):
 630                 sgd_vdu_mark = False
 631                 for vdu in get_iterable(indata.get("vdu")):
 632                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 633                         sgd_vdu_mark = True
 634                         break
 635                 if sgd_vdu_mark:
 636                     break
 637             else:
 638                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 639                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 640                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 641             for sca in get_iterable(sgd.get("scaling-config-action")):
 642                 if not indata.get("vnf-configuration"):
 643                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 644                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 645                                           .format(sgd["name"]),
 646                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 647                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 648                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 649                         break
 650                 else:
 651                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 652                                           "primitive-name-ref='{}' does not match any "
 653                                           "vnf-configuration:config-primitive:name"
 654                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 655                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 656         return indata
 657
 658     def _validate_input_edit(self, indata, force=False):
 659         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 660         return indata
 661
 662     def _validate_package_folders(self, storage_params, folder, file=None):
 663         if not storage_params or not storage_params.get("pkg-dir"):
 664             return False
 665         else:
 666             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 667                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 668             else:
 669                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 670             if file:
 671                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 672             else:
 673                 if self.fs.file_exists(f, 'dir'):
 674                     if self.fs.dir_ls(f):
 675                         return True
 676             return False
 677
 678
 679 class NsdTopic(DescriptorTopic):
 680     topic = "nsds"
 681     topic_msg = "nsd"
 682
 683     def __init__(self, db, fs, msg, auth):
 684         DescriptorTopic.__init__(self, db, fs, msg, auth)
 685
 686     @staticmethod
 687     def _remove_envelop(indata=None):
 688         if not indata:
 689             return {}
 690         clean_indata = indata
 691
 692         if clean_indata.get('nsd:nsd-catalog'):
 693             clean_indata = clean_indata['nsd:nsd-catalog']
 694         elif clean_indata.get('nsd-catalog'):
 695             clean_indata = clean_indata['nsd-catalog']
 696         if clean_indata.get('nsd'):
 697             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 698                 raise EngineException("'nsd' must be a list of only one element")
 699             clean_indata = clean_indata['nsd'][0]
 700         elif clean_indata.get('nsd:nsd'):
 701             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 702                 raise EngineException("'nsd:nsd' must be a list of only one element")
 703             clean_indata = clean_indata['nsd:nsd'][0]
 704         return clean_indata
 705
 706     def _validate_input_new(self, indata, storage_params, force=False):
 707         indata = self.pyangbind_validation("nsds", indata, force)
 708         # Cross references validation in the descriptor
 709         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 710         for vld in get_iterable(indata.get("vld")):
 711             if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
 712                 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
 713                                       " You cannot set an ip-profile when mgmt-network is True"
 714                                       .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 715             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 716                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 717                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 718                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 719                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 720                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 721                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 722                                                                  constituent_vnfd["member-vnf-index"],
 723                                                                  constituent_vnfd["vnfd-id-ref"]),
 724                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 725                         break
 726                 else:
 727                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 728                                           "does not match any constituent-vnfd:member-vnf-index"
 729                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 730                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 731         # Check VNFFGD
 732         for fgd in get_iterable(indata.get("vnffgd")):
 733             for cls in get_iterable(fgd.get("classifier")):
 734                 rspref = cls.get("rsp-id-ref")
 735                 for rsp in get_iterable(fgd.get("rsp")):
 736                     rspid = rsp.get("id")
 737                     if rspid and rspref and rspid == rspref:
 738                         break
 739                 else:
 740                     raise EngineException(
 741                         "Error at vnffgd[id='{}']:classifier[id='{}']:rsp-id-ref '{}' does not match any rsp:id"
 742                         .format(fgd["id"], cls["id"], rspref),
 743                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 744         return indata
 745
 746     def _validate_input_edit(self, indata, force=False):
 747         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 748         return indata
 749
 750     def _check_descriptor_dependencies(self, session, descriptor):
 751         """
 752         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 753         connection points are ok
 754         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 755         :param descriptor: descriptor to be inserted or edit
 756         :return: None or raises exception
 757         """
 758         if session["force"]:
 759             return
 760         member_vnfd_index = {}
 761         if descriptor.get("constituent-vnfd") and not session["force"]:
 762             for vnf in descriptor["constituent-vnfd"]:
 763                 vnfd_id = vnf["vnfd-id-ref"]
 764                 filter_q = self._get_project_filter(session)
 765                 filter_q["id"] = vnfd_id
 766                 vnf_list = self.db.get_list("vnfds", filter_q)
 767                 if not vnf_list:
 768                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 769                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 770                 # elif len(vnf_list) > 1:
 771                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 772                 #                           http_code=HTTPStatus.CONFLICT)
 773                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 774
 775         # Cross references validation in the descriptor and vnfd connection point validation
 776         for vld in get_iterable(descriptor.get("vld")):
 777             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 778                 # look if this vnfd contains this connection point
 779                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 780                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 781                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 782                         break
 783                 else:
 784                     raise EngineException(
 785                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 786                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 787                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 788                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 789                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 790
 791     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 792         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 793
 794         self._check_descriptor_dependencies(session, final_content)
 795
 796     def check_conflict_on_del(self, session, _id, db_content):
 797         """
 798         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 799         that NSD can be public and be used by other projects.
 800         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 801         :param _id: nsd internal id
 802         :param db_content: The database content of the _id
 803         :return: None or raises EngineException with the conflict
 804         """
 805         if session["force"]:
 806             return
 807         descriptor = db_content
 808         descriptor_id = descriptor.get("id")
 809         if not descriptor_id:  # empty nsd not uploaded
 810             return
 811
 812         # check NSD used by NS
 813         _filter = self._get_project_filter(session)
 814         _filter["nsd-id"] = _id
 815         if self.db.get_list("nsrs", _filter):
 816             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
 817
 818         # check NSD referenced by NST
 819         del _filter["nsd-id"]
 820         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
 821         if self.db.get_list("nsts", _filter):
 822             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
 823                                   http_code=HTTPStatus.CONFLICT)
 824
 825
 826 class NstTopic(DescriptorTopic):
 827     topic = "nsts"
 828     topic_msg = "nst"
 829
 830     def __init__(self, db, fs, msg, auth):
 831         DescriptorTopic.__init__(self, db, fs, msg, auth)
 832
 833     @staticmethod
 834     def _remove_envelop(indata=None):
 835         if not indata:
 836             return {}
 837         clean_indata = indata
 838
 839         if clean_indata.get('nst'):
 840             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 841                 raise EngineException("'nst' must be a list only one element")
 842             clean_indata = clean_indata['nst'][0]
 843         elif clean_indata.get('nst:nst'):
 844             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
 845                 raise EngineException("'nst:nst' must be a list only one element")
 846             clean_indata = clean_indata['nst:nst'][0]
 847         return clean_indata
 848
 849     def _validate_input_edit(self, indata, force=False):
 850         # TODO validate with pyangbind, serialize
 851         return indata
 852
 853     def _validate_input_new(self, indata, storage_params, force=False):
 854         indata = self.pyangbind_validation("nsts", indata, force)
 855         return indata.copy()
 856
 857     def _check_descriptor_dependencies(self, session, descriptor):
 858         """
 859         Check that the dependent descriptors exist on a new descriptor or edition
 860         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 861         :param descriptor: descriptor to be inserted or edit
 862         :return: None or raises exception
 863         """
 864         if not descriptor.get("netslice-subnet"):
 865             return
 866         for nsd in descriptor["netslice-subnet"]:
 867             nsd_id = nsd["nsd-ref"]
 868             filter_q = self._get_project_filter(session)
 869             filter_q["id"] = nsd_id
 870             if not self.db.get_list("nsds", filter_q):
 871                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
 872                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
 873
 874     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 875         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 876
 877         self._check_descriptor_dependencies(session, final_content)
 878
 879     def check_conflict_on_del(self, session, _id, db_content):
 880         """
 881         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
 882         that NST can be public and be used by other projects.
 883         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 884         :param _id: nst internal id
 885         :param db_content: The database content of the _id.
 886         :return: None or raises EngineException with the conflict
 887         """
 888         # TODO: Check this method
 889         if session["force"]:
 890             return
 891         # Get Network Slice Template from Database
 892         _filter = self._get_project_filter(session)
 893         _filter["_admin.nst-id"] = _id
 894         if self.db.get_list("nsis", _filter):
 895             raise EngineException("there is at least one Netslice Instance using this descriptor",
 896                                   http_code=HTTPStatus.CONFLICT)
 897
 898
 899 class PduTopic(BaseTopic):
 900     topic = "pdus"
 901     topic_msg = "pdu"
 902     schema_new = pdu_new_schema
 903     schema_edit = pdu_edit_schema
 904
 905     def __init__(self, db, fs, msg, auth):
 906         BaseTopic.__init__(self, db, fs, msg, auth)
 907
 908     @staticmethod
 909     def format_on_new(content, project_id=None, make_public=False):
 910         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 911         content["_admin"]["onboardingState"] = "CREATED"
 912         content["_admin"]["operationalState"] = "ENABLED"
 913         content["_admin"]["usageState"] = "NOT_IN_USE"
 914
 915     def check_conflict_on_del(self, session, _id, db_content):
 916         """
 917         Check that there is not any vnfr that uses this PDU
 918         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 919         :param _id: pdu internal id
 920         :param db_content: The database content of the _id.
 921         :return: None or raises EngineException with the conflict
 922         """
 923         if session["force"]:
 924             return
 925
 926         _filter = self._get_project_filter(session)
 927         _filter["vdur.pdu-id"] = _id
 928         if self.db.get_list("vnfrs", _filter):
 929             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)