osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from time import time
  24 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema
  25 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  26 from osm_im.vnfd import vnfd as vnfd_im
  27 from osm_im.nsd import nsd as nsd_im
  28 from osm_im.nst import nst as nst_im
  29 from pyangbind.lib.serialise import pybindJSONDecoder
  30 import pyangbind.lib.pybindJSON as pybindJSON
  31
  32 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  33
  34
  35 class DescriptorTopic(BaseTopic):
  36
  37     def __init__(self, db, fs, msg, auth):
  38         BaseTopic.__init__(self, db, fs, msg, auth)
  39
  40     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  41         super().check_conflict_on_edit(session, final_content, edit_content, _id)
  42
  43         def _check_unique_id_name(descriptor, position=""):
  44             for desc_key, desc_item in descriptor.items():
  45                 if isinstance(desc_item, list) and desc_item:
  46                     used_ids = []
  47                     desc_item_id = None
  48                     for index, list_item in enumerate(desc_item):
  49                         if isinstance(list_item, dict):
  50                             _check_unique_id_name(list_item, "{}.{}[{}]"
  51                                                   .format(position, desc_key, index))
  52                             # Base case
  53                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  54                                 desc_item_id = "id" if list_item.get("id") else "name"
  55                             if desc_item_id and list_item.get(desc_item_id):
  56                                 if list_item[desc_item_id] in used_ids:
  57                                     position = "{}.{}[{}]".format(position, desc_key, index)
  58                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  59                                                           .format(desc_item_id, list_item[desc_item_id],
  60                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  61                                 used_ids.append(list_item[desc_item_id])
  62         _check_unique_id_name(final_content)
  63         # 1. validate again with pyangbind
  64         # 1.1. remove internal keys
  65         internal_keys = {}
  66         for k in ("_id", "_admin"):
  67             if k in final_content:
  68                 internal_keys[k] = final_content.pop(k)
  69         storage_params = internal_keys["_admin"].get("storage")
  70         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  71         # 1.2. modify final_content with a serialized version
  72         final_content.clear()
  73         final_content.update(serialized)
  74         # 1.3. restore internal keys
  75         for k, v in internal_keys.items():
  76             final_content[k] = v
  77
  78         if session["force"]:
  79             return
  80         # 2. check that this id is not present
  81         if "id" in edit_content:
  82             _filter = self._get_project_filter(session)
  83             _filter["id"] = final_content["id"]
  84             _filter["_id.neq"] = _id
  85             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  86                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  87                                                                                                final_content["id"]),
  88                                       HTTPStatus.CONFLICT)
  89
  90     @staticmethod
  91     def format_on_new(content, project_id=None, make_public=False):
  92         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  93         content["_admin"]["onboardingState"] = "CREATED"
  94         content["_admin"]["operationalState"] = "DISABLED"
  95         content["_admin"]["usageState"] = "NOT_IN_USE"
  96
  97     def delete_extra(self, session, _id, db_content, not_send_msg=None):
  98         """
  99         Deletes file system storage associated with the descriptor
 100         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 101         :param _id: server internal id
 102         :param db_content: The database content of the descriptor
 103         :param not_send_msg: To not send message (False) or store content (list) instead
 104         :return: None if ok or raises EngineException with the problem
 105         """
 106         self.fs.file_delete(_id, ignore_non_exist=True)
 107         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 108
 109     @staticmethod
 110     def get_one_by_id(db, session, topic, id):
 111         # find owned by this project
 112         _filter = BaseTopic._get_project_filter(session)
 113         _filter["id"] = id
 114         desc_list = db.get_list(topic, _filter)
 115         if len(desc_list) == 1:
 116             return desc_list[0]
 117         elif len(desc_list) > 1:
 118             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 119                               HTTPStatus.CONFLICT)
 120
 121         # not found any: try to find public
 122         _filter = BaseTopic._get_project_filter(session)
 123         _filter["id"] = id
 124         desc_list = db.get_list(topic, _filter)
 125         if not desc_list:
 126             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 127         elif len(desc_list) == 1:
 128             return desc_list[0]
 129         else:
 130             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 131                 topic[:-1], id), HTTPStatus.CONFLICT)
 132
 133     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 134         """
 135         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 136         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 137         (self.upload_content)
 138         :param rollback: list to append created items at database in case a rollback may to be done
 139         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 140         :param indata: data to be inserted
 141         :param kwargs: used to override the indata descriptor
 142         :param headers: http request headers
 143         :return: _id, None: identity of the inserted data; and None as there is not any operation
 144         """
 145
 146         try:
 147             # Check Quota
 148             self.check_quota(session)
 149
 150             # _remove_envelop
 151             if indata:
 152                 if "userDefinedData" in indata:
 153                     indata = indata['userDefinedData']
 154
 155             # Override descriptor with query string kwargs
 156             self._update_input_with_kwargs(indata, kwargs)
 157             # uncomment when this method is implemented.
 158             # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 159             # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 160
 161             content = {"_admin": {"userDefinedData": indata}}
 162             self.format_on_new(content, session["project_id"], make_public=session["public"])
 163             _id = self.db.create(self.topic, content)
 164             rollback.append({"topic": self.topic, "_id": _id})
 165             self._send_msg("created", {"_id": _id})
 166             return _id, None
 167         except ValidationError as e:
 168             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 169
 170     def upload_content(self, session, _id, indata, kwargs, headers):
 171         """
 172         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 173         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 174         :param _id : the nsd,vnfd is already created, this is the id
 175         :param indata: http body request
 176         :param kwargs: user query string to override parameters. NOT USED
 177         :param headers:  http request headers
 178         :return: True if package is completely uploaded or False if partial content has been uploded
 179             Raise exception on error
 180         """
 181         # Check that _id exists and it is valid
 182         current_desc = self.show(session, _id)
 183
 184         content_range_text = headers.get("Content-Range")
 185         expected_md5 = headers.get("Content-File-MD5")
 186         compressed = None
 187         content_type = headers.get("Content-Type")
 188         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 189                 "application/zip" in content_type:
 190             compressed = "gzip"
 191         filename = headers.get("Content-Filename")
 192         if not filename:
 193             filename = "package.tar.gz" if compressed else "package"
 194         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 195         file_pkg = None
 196         error_text = ""
 197         try:
 198             if content_range_text:
 199                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 200                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 201                     raise IndexError()
 202                 start = int(content_range[1])
 203                 end = int(content_range[2]) + 1
 204                 total = int(content_range[3])
 205             else:
 206                 start = 0
 207             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 208
 209             if start:
 210                 if not self.fs.file_exists(temp_folder, 'dir'):
 211                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 212             else:
 213                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 214                 self.fs.mkdir(temp_folder)
 215
 216             storage = self.fs.get_params()
 217             storage["folder"] = _id
 218
 219             file_path = (temp_folder, filename)
 220             if self.fs.file_exists(file_path, 'file'):
 221                 file_size = self.fs.file_size(file_path)
 222             else:
 223                 file_size = 0
 224             if file_size != start:
 225                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 226                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 227             file_pkg = self.fs.file_open(file_path, 'a+b')
 228             if isinstance(indata, dict):
 229                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 230                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 231             else:
 232                 indata_len = 0
 233                 while True:
 234                     indata_text = indata.read(4096)
 235                     indata_len += len(indata_text)
 236                     if not indata_text:
 237                         break
 238                     file_pkg.write(indata_text)
 239             if content_range_text:
 240                 if indata_len != end-start:
 241                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 242                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 243                 if end != total:
 244                     # TODO update to UPLOADING
 245                     return False
 246
 247             # PACKAGE UPLOADED
 248             if expected_md5:
 249                 file_pkg.seek(0, 0)
 250                 file_md5 = md5()
 251                 chunk_data = file_pkg.read(1024)
 252                 while chunk_data:
 253                     file_md5.update(chunk_data)
 254                     chunk_data = file_pkg.read(1024)
 255                 if expected_md5 != file_md5.hexdigest():
 256                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 257             file_pkg.seek(0, 0)
 258             if compressed == "gzip":
 259                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 260                 descriptor_file_name = None
 261                 for tarinfo in tar:
 262                     tarname = tarinfo.name
 263                     tarname_path = tarname.split("/")
 264                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 265                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 266                     if len(tarname_path) == 1 and not tarinfo.isdir():
 267                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 268                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 269                         storage["pkg-dir"] = tarname_path[0]
 270                         if len(tarname_path) == 2:
 271                             if descriptor_file_name:
 272                                 raise EngineException(
 273                                     "Found more than one descriptor file at package descriptor tar.gz")
 274                             descriptor_file_name = tarname
 275                 if not descriptor_file_name:
 276                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 277                 storage["descriptor"] = descriptor_file_name
 278                 storage["zipfile"] = filename
 279                 self.fs.file_extract(tar, temp_folder)
 280                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 281                     content = descriptor_file.read()
 282             else:
 283                 content = file_pkg.read()
 284                 storage["descriptor"] = descriptor_file_name = filename
 285
 286             if descriptor_file_name.endswith(".json"):
 287                 error_text = "Invalid json format "
 288                 indata = json.load(content)
 289             else:
 290                 error_text = "Invalid yaml format "
 291                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 292
 293             current_desc["_admin"]["storage"] = storage
 294             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 295             current_desc["_admin"]["operationalState"] = "ENABLED"
 296
 297             indata = self._remove_envelop(indata)
 298
 299             # Override descriptor with query string kwargs
 300             if kwargs:
 301                 self._update_input_with_kwargs(indata, kwargs)
 302             # it will call overrides method at VnfdTopic or NsdTopic
 303             # indata = self._validate_input_edit(indata, force=session["force"])
 304
 305             deep_update_rfc7396(current_desc, indata)
 306             self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 307             current_desc["_admin"]["modified"] = time()
 308             self.db.replace(self.topic, _id, current_desc)
 309             self.fs.dir_rename(temp_folder, _id)
 310
 311             indata["_id"] = _id
 312             self._send_msg("edited", indata)
 313
 314             # TODO if descriptor has changed because kwargs update content and remove cached zip
 315             # TODO if zip is not present creates one
 316             return True
 317
 318         except EngineException:
 319             raise
 320         except IndexError:
 321             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 322                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 323         except IOError as e:
 324             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 325         except tarfile.ReadError as e:
 326             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 327         except (ValueError, yaml.YAMLError) as e:
 328             raise EngineException(error_text + str(e))
 329         except ValidationError as e:
 330             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 331         finally:
 332             if file_pkg:
 333                 file_pkg.close()
 334
 335     def get_file(self, session, _id, path=None, accept_header=None):
 336         """
 337         Return the file content of a vnfd or nsd
 338         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 339         :param _id: Identity of the vnfd, nsd
 340         :param path: artifact path or "$DESCRIPTOR" or None
 341         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 342         :return: opened file plus Accept format or raises an exception
 343         """
 344         accept_text = accept_zip = False
 345         if accept_header:
 346             if 'text/plain' in accept_header or '*/*' in accept_header:
 347                 accept_text = True
 348             if 'application/zip' in accept_header or '*/*' in accept_header:
 349                 accept_zip = 'application/zip'
 350             elif 'application/gzip' in accept_header:
 351                 accept_zip = 'application/gzip'
 352
 353         if not accept_text and not accept_zip:
 354             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 355                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 356
 357         content = self.show(session, _id)
 358         if content["_admin"]["onboardingState"] != "ONBOARDED":
 359             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 360                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 361                                   http_code=HTTPStatus.CONFLICT)
 362         storage = content["_admin"]["storage"]
 363         if path is not None and path != "$DESCRIPTOR":   # artifacts
 364             if not storage.get('pkg-dir'):
 365                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 366             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 367                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 368                 return folder_content, "text/plain"
 369                 # TODO manage folders in http
 370             else:
 371                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 372                     "application/octet-stream"
 373
 374         # pkgtype   accept  ZIP  TEXT    -> result
 375         # manyfiles         yes  X       -> zip
 376         #                   no   yes     -> error
 377         # onefile           yes  no      -> zip
 378         #                   X    yes     -> text
 379
 380         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 381             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 382         elif storage.get('pkg-dir') and not accept_zip:
 383             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 384                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 385         else:
 386             if not storage.get('zipfile'):
 387                 # TODO generate zipfile if not present
 388                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 389                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 390             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 391
 392     def pyangbind_validation(self, item, data, force=False):
 393         try:
 394             if item == "vnfds":
 395                 myvnfd = vnfd_im()
 396                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 397                                                  path_helper=True, skip_unknown=force)
 398                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 399             elif item == "nsds":
 400                 mynsd = nsd_im()
 401                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 402                                                  path_helper=True, skip_unknown=force)
 403                 out = pybindJSON.dumps(mynsd, mode="ietf")
 404             elif item == "nsts":
 405                 mynst = nst_im()
 406                 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
 407                                                  path_helper=True, skip_unknown=force)
 408                 out = pybindJSON.dumps(mynst, mode="ietf")
 409             else:
 410                 raise EngineException("Not possible to validate '{}' item".format(item),
 411                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 412
 413             desc_out = self._remove_envelop(yaml.safe_load(out))
 414             return desc_out
 415
 416         except Exception as e:
 417             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 418                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 419
 420
 421 class VnfdTopic(DescriptorTopic):
 422     topic = "vnfds"
 423     topic_msg = "vnfd"
 424
 425     def __init__(self, db, fs, msg, auth):
 426         DescriptorTopic.__init__(self, db, fs, msg, auth)
 427
 428     @staticmethod
 429     def _remove_envelop(indata=None):
 430         if not indata:
 431             return {}
 432         clean_indata = indata
 433         if clean_indata.get('vnfd:vnfd-catalog'):
 434             clean_indata = clean_indata['vnfd:vnfd-catalog']
 435         elif clean_indata.get('vnfd-catalog'):
 436             clean_indata = clean_indata['vnfd-catalog']
 437         if clean_indata.get('vnfd'):
 438             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 439                 raise EngineException("'vnfd' must be a list of only one element")
 440             clean_indata = clean_indata['vnfd'][0]
 441         elif clean_indata.get('vnfd:vnfd'):
 442             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 443                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 444             clean_indata = clean_indata['vnfd:vnfd'][0]
 445         return clean_indata
 446
 447     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 448         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 449
 450         # set type of vnfd
 451         contains_pdu = False
 452         contains_vdu = False
 453         for vdu in get_iterable(final_content.get("vdu")):
 454             if vdu.get("pdu-type"):
 455                 contains_pdu = True
 456             else:
 457                 contains_vdu = True
 458         if contains_pdu:
 459             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 460         elif contains_vdu:
 461             final_content["_admin"]["type"] = "vnfd"
 462         # if neither vud nor pdu do not fill type
 463
 464     def check_conflict_on_del(self, session, _id, db_content):
 465         """
 466         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 467         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 468         that uses this vnfd
 469         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 470         :param _id: vnfd internal id
 471         :param db_content: The database content of the _id.
 472         :return: None or raises EngineException with the conflict
 473         """
 474         if session["force"]:
 475             return
 476         descriptor = db_content
 477         descriptor_id = descriptor.get("id")
 478         if not descriptor_id:  # empty vnfd not uploaded
 479             return
 480
 481         _filter = self._get_project_filter(session)
 482
 483         # check vnfrs using this vnfd
 484         _filter["vnfd-id"] = _id
 485         if self.db.get_list("vnfrs", _filter):
 486             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 487
 488         # check NSD referencing this VNFD
 489         del _filter["vnfd-id"]
 490         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 491         if self.db.get_list("nsds", _filter):
 492             raise EngineException("There is at least one NSD referencing this descriptor",
 493                                   http_code=HTTPStatus.CONFLICT)
 494
 495     def _validate_input_new(self, indata, storage_params, force=False):
 496         indata = self.pyangbind_validation("vnfds", indata, force)
 497         # Cross references validation in the descriptor
 498         if indata.get("vdu"):
 499             if not indata.get("mgmt-interface"):
 500                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 501                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 502             if indata["mgmt-interface"].get("cp"):
 503                 for cp in get_iterable(indata.get("connection-point")):
 504                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 505                         break
 506                 else:
 507                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 508                                           .format(indata["mgmt-interface"]["cp"]),
 509                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 510
 511         for vdu in get_iterable(indata.get("vdu")):
 512             icp_refs = []
 513             ecp_refs = []
 514             for interface in get_iterable(vdu.get("interface")):
 515                 if interface.get("external-connection-point-ref"):
 516                     if interface.get("external-connection-point-ref") in ecp_refs:
 517                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 518                                               "is referenced by other interface"
 519                                               .format(vdu["id"], interface["name"],
 520                                                       interface["external-connection-point-ref"]),
 521                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 522                     ecp_refs.append(interface.get("external-connection-point-ref"))
 523                     for cp in get_iterable(indata.get("connection-point")):
 524                         if cp["name"] == interface["external-connection-point-ref"]:
 525                             break
 526                     else:
 527                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 528                                               "must match an existing connection-point"
 529                                               .format(vdu["id"], interface["name"],
 530                                                       interface["external-connection-point-ref"]),
 531                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 532                 elif interface.get("internal-connection-point-ref"):
 533                     if interface.get("internal-connection-point-ref") in icp_refs:
 534                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 535                                               "is referenced by other interface"
 536                                               .format(vdu["id"], interface["name"],
 537                                                       interface["internal-connection-point-ref"]),
 538                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 539                     icp_refs.append(interface.get("internal-connection-point-ref"))
 540                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 541                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 542                             break
 543                     else:
 544                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 545                                               "must match an existing vdu:internal-connection-point"
 546                                               .format(vdu["id"], interface["name"],
 547                                                       interface["internal-connection-point-ref"]),
 548                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 549             # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 550             if vdu.get("vdu-configuration"):
 551                 if vdu["vdu-configuration"].get("juju"):
 552                     if not self._validate_package_folders(storage_params, 'charms'):
 553                         raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
 554                                               "package".format(indata["id"], vdu["id"]))
 555             # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
 556             if vdu.get("cloud-init-file"):
 557                 if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 558                     raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 559                                           "package".format(indata["id"], vdu["id"]))
 560         # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 561         if indata.get("vnf-configuration"):
 562             if indata["vnf-configuration"].get("juju"):
 563                 if not self._validate_package_folders(storage_params, 'charms'):
 564                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 565                                           "package".format(indata["id"]))
 566         vld_names = []  # For detection of duplicated VLD names
 567         for ivld in get_iterable(indata.get("internal-vld")):
 568             # BEGIN Detection of duplicated VLD names
 569             ivld_name = ivld["name"]
 570             if ivld_name in vld_names:
 571                 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
 572                                       .format(ivld["name"], indata["id"], ivld["id"]),
 573                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 574             else:
 575                 vld_names.append(ivld_name)
 576             # END Detection of duplicated VLD names
 577             for icp in get_iterable(ivld.get("internal-connection-point")):
 578                 icp_mark = False
 579                 for vdu in get_iterable(indata.get("vdu")):
 580                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 581                         if icp["id-ref"] == internal_cp["id"]:
 582                             icp_mark = True
 583                             break
 584                     if icp_mark:
 585                         break
 586                 else:
 587                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 588                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 589                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 590             if ivld.get("ip-profile-ref"):
 591                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 592                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 593                         break
 594                 else:
 595                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 596                         ivld["id"], ivld["ip-profile-ref"]),
 597                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 598         for mp in get_iterable(indata.get("monitoring-param")):
 599             if mp.get("vdu-monitoring-param"):
 600                 mp_vmp_mark = False
 601                 for vdu in get_iterable(indata.get("vdu")):
 602                     for vmp in get_iterable(vdu.get("monitoring-param")):
 603                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 604                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 605                             mp_vmp_mark = True
 606                             break
 607                     if mp_vmp_mark:
 608                         break
 609                 else:
 610                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 611                                           "defined at vdu[id='{}'] or vdu does not exist"
 612                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 613                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 614                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 615             elif mp.get("vdu-metric"):
 616                 mp_vm_mark = False
 617                 for vdu in get_iterable(indata.get("vdu")):
 618                     if vdu.get("vdu-configuration"):
 619                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 620                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 621                                     mp["vdu-metric"]["vdu-ref"]:
 622                                 mp_vm_mark = True
 623                                 break
 624                         if mp_vm_mark:
 625                             break
 626                 else:
 627                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 628                                           "vdu[id='{}'] or vdu does not exist"
 629                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 630                                                   mp["vdu-metric"]["vdu-ref"]),
 631                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 632
 633         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 634             for sp in get_iterable(sgd.get("scaling-policy")):
 635                 for sc in get_iterable(sp.get("scaling-criteria")):
 636                     for mp in get_iterable(indata.get("monitoring-param")):
 637                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 638                             break
 639                     else:
 640                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 641                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 642                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 643                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 644             for sgd_vdu in get_iterable(sgd.get("vdu")):
 645                 sgd_vdu_mark = False
 646                 for vdu in get_iterable(indata.get("vdu")):
 647                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 648                         sgd_vdu_mark = True
 649                         break
 650                 if sgd_vdu_mark:
 651                     break
 652             else:
 653                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 654                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 655                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 656             for sca in get_iterable(sgd.get("scaling-config-action")):
 657                 if not indata.get("vnf-configuration"):
 658                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 659                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 660                                           .format(sgd["name"]),
 661                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 662                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 663                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 664                         break
 665                 else:
 666                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 667                                           "primitive-name-ref='{}' does not match any "
 668                                           "vnf-configuration:config-primitive:name"
 669                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 670                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 671         return indata
 672
 673     def _validate_input_edit(self, indata, force=False):
 674         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 675         return indata
 676
 677     def _validate_package_folders(self, storage_params, folder, file=None):
 678         if not storage_params or not storage_params.get("pkg-dir"):
 679             return False
 680         else:
 681             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 682                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 683             else:
 684                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 685             if file:
 686                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 687             else:
 688                 if self.fs.file_exists(f, 'dir'):
 689                     if self.fs.dir_ls(f):
 690                         return True
 691             return False
 692
 693
 694 class NsdTopic(DescriptorTopic):
 695     topic = "nsds"
 696     topic_msg = "nsd"
 697
 698     def __init__(self, db, fs, msg, auth):
 699         DescriptorTopic.__init__(self, db, fs, msg, auth)
 700
 701     @staticmethod
 702     def _remove_envelop(indata=None):
 703         if not indata:
 704             return {}
 705         clean_indata = indata
 706
 707         if clean_indata.get('nsd:nsd-catalog'):
 708             clean_indata = clean_indata['nsd:nsd-catalog']
 709         elif clean_indata.get('nsd-catalog'):
 710             clean_indata = clean_indata['nsd-catalog']
 711         if clean_indata.get('nsd'):
 712             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 713                 raise EngineException("'nsd' must be a list of only one element")
 714             clean_indata = clean_indata['nsd'][0]
 715         elif clean_indata.get('nsd:nsd'):
 716             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 717                 raise EngineException("'nsd:nsd' must be a list of only one element")
 718             clean_indata = clean_indata['nsd:nsd'][0]
 719         return clean_indata
 720
 721     def _validate_input_new(self, indata, storage_params, force=False):
 722         indata = self.pyangbind_validation("nsds", indata, force)
 723         # Cross references validation in the descriptor
 724         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 725         for vld in get_iterable(indata.get("vld")):
 726             if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
 727                 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
 728                                       " You cannot set an ip-profile when mgmt-network is True"
 729                                       .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 730             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 731                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 732                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 733                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 734                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 735                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 736                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 737                                                                  constituent_vnfd["member-vnf-index"],
 738                                                                  constituent_vnfd["vnfd-id-ref"]),
 739                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 740                         break
 741                 else:
 742                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 743                                           "does not match any constituent-vnfd:member-vnf-index"
 744                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 745                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 746         # Check VNFFGD
 747         for fgd in get_iterable(indata.get("vnffgd")):
 748             for cls in get_iterable(fgd.get("classifier")):
 749                 rspref = cls.get("rsp-id-ref")
 750                 for rsp in get_iterable(fgd.get("rsp")):
 751                     rspid = rsp.get("id")
 752                     if rspid and rspref and rspid == rspref:
 753                         break
 754                 else:
 755                     raise EngineException(
 756                         "Error at vnffgd[id='{}']:classifier[id='{}']:rsp-id-ref '{}' does not match any rsp:id"
 757                         .format(fgd["id"], cls["id"], rspref),
 758                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 759         return indata
 760
 761     def _validate_input_edit(self, indata, force=False):
 762         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 763         return indata
 764
 765     def _check_descriptor_dependencies(self, session, descriptor):
 766         """
 767         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 768         connection points are ok
 769         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 770         :param descriptor: descriptor to be inserted or edit
 771         :return: None or raises exception
 772         """
 773         if session["force"]:
 774             return
 775         member_vnfd_index = {}
 776         if descriptor.get("constituent-vnfd") and not session["force"]:
 777             for vnf in descriptor["constituent-vnfd"]:
 778                 vnfd_id = vnf["vnfd-id-ref"]
 779                 filter_q = self._get_project_filter(session)
 780                 filter_q["id"] = vnfd_id
 781                 vnf_list = self.db.get_list("vnfds", filter_q)
 782                 if not vnf_list:
 783                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 784                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 785                 # elif len(vnf_list) > 1:
 786                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 787                 #                           http_code=HTTPStatus.CONFLICT)
 788                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 789
 790         # Cross references validation in the descriptor and vnfd connection point validation
 791         for vld in get_iterable(descriptor.get("vld")):
 792             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 793                 # look if this vnfd contains this connection point
 794                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 795                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 796                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 797                         break
 798                 else:
 799                     raise EngineException(
 800                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 801                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 802                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 803                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 804                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 805
 806     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 807         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 808
 809         self._check_descriptor_dependencies(session, final_content)
 810
 811     def check_conflict_on_del(self, session, _id, db_content):
 812         """
 813         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 814         that NSD can be public and be used by other projects.
 815         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 816         :param _id: nsd internal id
 817         :param db_content: The database content of the _id
 818         :return: None or raises EngineException with the conflict
 819         """
 820         if session["force"]:
 821             return
 822         descriptor = db_content
 823         descriptor_id = descriptor.get("id")
 824         if not descriptor_id:  # empty nsd not uploaded
 825             return
 826
 827         # check NSD used by NS
 828         _filter = self._get_project_filter(session)
 829         _filter["nsd-id"] = _id
 830         if self.db.get_list("nsrs", _filter):
 831             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
 832
 833         # check NSD referenced by NST
 834         del _filter["nsd-id"]
 835         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
 836         if self.db.get_list("nsts", _filter):
 837             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
 838                                   http_code=HTTPStatus.CONFLICT)
 839
 840
 841 class NstTopic(DescriptorTopic):
 842     topic = "nsts"
 843     topic_msg = "nst"
 844
 845     def __init__(self, db, fs, msg, auth):
 846         DescriptorTopic.__init__(self, db, fs, msg, auth)
 847
 848     @staticmethod
 849     def _remove_envelop(indata=None):
 850         if not indata:
 851             return {}
 852         clean_indata = indata
 853
 854         if clean_indata.get('nst'):
 855             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 856                 raise EngineException("'nst' must be a list only one element")
 857             clean_indata = clean_indata['nst'][0]
 858         elif clean_indata.get('nst:nst'):
 859             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
 860                 raise EngineException("'nst:nst' must be a list only one element")
 861             clean_indata = clean_indata['nst:nst'][0]
 862         return clean_indata
 863
 864     def _validate_input_edit(self, indata, force=False):
 865         # TODO validate with pyangbind, serialize
 866         return indata
 867
 868     def _validate_input_new(self, indata, storage_params, force=False):
 869         indata = self.pyangbind_validation("nsts", indata, force)
 870         return indata.copy()
 871
 872     def _check_descriptor_dependencies(self, session, descriptor):
 873         """
 874         Check that the dependent descriptors exist on a new descriptor or edition
 875         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 876         :param descriptor: descriptor to be inserted or edit
 877         :return: None or raises exception
 878         """
 879         if not descriptor.get("netslice-subnet"):
 880             return
 881         for nsd in descriptor["netslice-subnet"]:
 882             nsd_id = nsd["nsd-ref"]
 883             filter_q = self._get_project_filter(session)
 884             filter_q["id"] = nsd_id
 885             if not self.db.get_list("nsds", filter_q):
 886                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
 887                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
 888
 889     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 890         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 891
 892         self._check_descriptor_dependencies(session, final_content)
 893
 894     def check_conflict_on_del(self, session, _id, db_content):
 895         """
 896         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
 897         that NST can be public and be used by other projects.
 898         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 899         :param _id: nst internal id
 900         :param db_content: The database content of the _id.
 901         :return: None or raises EngineException with the conflict
 902         """
 903         # TODO: Check this method
 904         if session["force"]:
 905             return
 906         # Get Network Slice Template from Database
 907         _filter = self._get_project_filter(session)
 908         _filter["_admin.nst-id"] = _id
 909         if self.db.get_list("nsis", _filter):
 910             raise EngineException("there is at least one Netslice Instance using this descriptor",
 911                                   http_code=HTTPStatus.CONFLICT)
 912
 913
 914 class PduTopic(BaseTopic):
 915     topic = "pdus"
 916     topic_msg = "pdu"
 917     schema_new = pdu_new_schema
 918     schema_edit = pdu_edit_schema
 919
 920     def __init__(self, db, fs, msg, auth):
 921         BaseTopic.__init__(self, db, fs, msg, auth)
 922
 923     @staticmethod
 924     def format_on_new(content, project_id=None, make_public=False):
 925         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 926         content["_admin"]["onboardingState"] = "CREATED"
 927         content["_admin"]["operationalState"] = "ENABLED"
 928         content["_admin"]["usageState"] = "NOT_IN_USE"
 929
 930     def check_conflict_on_del(self, session, _id, db_content):
 931         """
 932         Check that there is not any vnfr that uses this PDU
 933         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 934         :param _id: pdu internal id
 935         :param db_content: The database content of the _id.
 936         :return: None or raises EngineException with the conflict
 937         """
 938         if session["force"]:
 939             return
 940
 941         _filter = self._get_project_filter(session)
 942         _filter["vdur.pdu-id"] = _id
 943         if self.db.get_list("vnfrs", _filter):
 944             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)