osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from time import time
  24 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema
  25 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  26 from osm_im.vnfd import vnfd as vnfd_im
  27 from osm_im.nsd import nsd as nsd_im
  28 from osm_im.nst import nst as nst_im
  29 from pyangbind.lib.serialise import pybindJSONDecoder
  30 import pyangbind.lib.pybindJSON as pybindJSON
  31
  32 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  33
  34
  35 class DescriptorTopic(BaseTopic):
  36
  37     def __init__(self, db, fs, msg, auth):
  38         BaseTopic.__init__(self, db, fs, msg, auth)
  39
  40     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  41         super().check_conflict_on_edit(session, final_content, edit_content, _id)
  42
  43         def _check_unique_id_name(descriptor, position=""):
  44             for desc_key, desc_item in descriptor.items():
  45                 if isinstance(desc_item, list) and desc_item:
  46                     used_ids = []
  47                     desc_item_id = None
  48                     for index, list_item in enumerate(desc_item):
  49                         if isinstance(list_item, dict):
  50                             _check_unique_id_name(list_item, "{}.{}[{}]"
  51                                                   .format(position, desc_key, index))
  52                             # Base case
  53                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  54                                 desc_item_id = "id" if list_item.get("id") else "name"
  55                             if desc_item_id and list_item.get(desc_item_id):
  56                                 if list_item[desc_item_id] in used_ids:
  57                                     position = "{}.{}[{}]".format(position, desc_key, index)
  58                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  59                                                           .format(desc_item_id, list_item[desc_item_id],
  60                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  61                                 used_ids.append(list_item[desc_item_id])
  62         _check_unique_id_name(final_content)
  63         # 1. validate again with pyangbind
  64         # 1.1. remove internal keys
  65         internal_keys = {}
  66         for k in ("_id", "_admin"):
  67             if k in final_content:
  68                 internal_keys[k] = final_content.pop(k)
  69         storage_params = internal_keys["_admin"].get("storage")
  70         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  71         # 1.2. modify final_content with a serialized version
  72         final_content.clear()
  73         final_content.update(serialized)
  74         # 1.3. restore internal keys
  75         for k, v in internal_keys.items():
  76             final_content[k] = v
  77
  78         if session["force"]:
  79             return
  80         # 2. check that this id is not present
  81         if "id" in edit_content:
  82             _filter = self._get_project_filter(session)
  83             _filter["id"] = final_content["id"]
  84             _filter["_id.neq"] = _id
  85             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  86                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  87                                                                                                final_content["id"]),
  88                                       HTTPStatus.CONFLICT)
  89
  90     @staticmethod
  91     def format_on_new(content, project_id=None, make_public=False):
  92         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  93         content["_admin"]["onboardingState"] = "CREATED"
  94         content["_admin"]["operationalState"] = "DISABLED"
  95         content["_admin"]["usageState"] = "NOT_IN_USE"
  96
  97     def delete_extra(self, session, _id, db_content):
  98         """
  99         Deletes file system storage associated with the descriptor
 100         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 101         :param _id: server internal id
 102         :param db_content: The database content of the descriptor
 103         :return: None if ok or raises EngineException with the problem
 104         """
 105         self.fs.file_delete(_id, ignore_non_exist=True)
 106         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 107
 108     @staticmethod
 109     def get_one_by_id(db, session, topic, id):
 110         # find owned by this project
 111         _filter = BaseTopic._get_project_filter(session)
 112         _filter["id"] = id
 113         desc_list = db.get_list(topic, _filter)
 114         if len(desc_list) == 1:
 115             return desc_list[0]
 116         elif len(desc_list) > 1:
 117             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 118                               HTTPStatus.CONFLICT)
 119
 120         # not found any: try to find public
 121         _filter = BaseTopic._get_project_filter(session)
 122         _filter["id"] = id
 123         desc_list = db.get_list(topic, _filter)
 124         if not desc_list:
 125             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 126         elif len(desc_list) == 1:
 127             return desc_list[0]
 128         else:
 129             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 130                 topic[:-1], id), HTTPStatus.CONFLICT)
 131
 132     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 133         """
 134         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 135         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 136         (self.upload_content)
 137         :param rollback: list to append created items at database in case a rollback may to be done
 138         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 139         :param indata: data to be inserted
 140         :param kwargs: used to override the indata descriptor
 141         :param headers: http request headers
 142         :return: _id, None: identity of the inserted data; and None as there is not any operation
 143         """
 144
 145         try:
 146             # Check Quota
 147             self.check_quota(session)
 148
 149             # _remove_envelop
 150             if indata:
 151                 if "userDefinedData" in indata:
 152                     indata = indata['userDefinedData']
 153
 154             # Override descriptor with query string kwargs
 155             self._update_input_with_kwargs(indata, kwargs)
 156             # uncomment when this method is implemented.
 157             # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 158             # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 159
 160             content = {"_admin": {"userDefinedData": indata}}
 161             self.format_on_new(content, session["project_id"], make_public=session["public"])
 162             _id = self.db.create(self.topic, content)
 163             rollback.append({"topic": self.topic, "_id": _id})
 164             self._send_msg("created", {"_id": _id})
 165             return _id, None
 166         except ValidationError as e:
 167             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 168
 169     def upload_content(self, session, _id, indata, kwargs, headers):
 170         """
 171         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 172         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 173         :param _id : the nsd,vnfd is already created, this is the id
 174         :param indata: http body request
 175         :param kwargs: user query string to override parameters. NOT USED
 176         :param headers:  http request headers
 177         :return: True if package is completely uploaded or False if partial content has been uploded
 178             Raise exception on error
 179         """
 180         # Check that _id exists and it is valid
 181         current_desc = self.show(session, _id)
 182
 183         content_range_text = headers.get("Content-Range")
 184         expected_md5 = headers.get("Content-File-MD5")
 185         compressed = None
 186         content_type = headers.get("Content-Type")
 187         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 188                 "application/zip" in content_type:
 189             compressed = "gzip"
 190         filename = headers.get("Content-Filename")
 191         if not filename:
 192             filename = "package.tar.gz" if compressed else "package"
 193         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 194         file_pkg = None
 195         error_text = ""
 196         try:
 197             if content_range_text:
 198                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 199                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 200                     raise IndexError()
 201                 start = int(content_range[1])
 202                 end = int(content_range[2]) + 1
 203                 total = int(content_range[3])
 204             else:
 205                 start = 0
 206             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 207
 208             if start:
 209                 if not self.fs.file_exists(temp_folder, 'dir'):
 210                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 211             else:
 212                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 213                 self.fs.mkdir(temp_folder)
 214
 215             storage = self.fs.get_params()
 216             storage["folder"] = _id
 217
 218             file_path = (temp_folder, filename)
 219             if self.fs.file_exists(file_path, 'file'):
 220                 file_size = self.fs.file_size(file_path)
 221             else:
 222                 file_size = 0
 223             if file_size != start:
 224                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 225                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 226             file_pkg = self.fs.file_open(file_path, 'a+b')
 227             if isinstance(indata, dict):
 228                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 229                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 230             else:
 231                 indata_len = 0
 232                 while True:
 233                     indata_text = indata.read(4096)
 234                     indata_len += len(indata_text)
 235                     if not indata_text:
 236                         break
 237                     file_pkg.write(indata_text)
 238             if content_range_text:
 239                 if indata_len != end-start:
 240                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 241                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 242                 if end != total:
 243                     # TODO update to UPLOADING
 244                     return False
 245
 246             # PACKAGE UPLOADED
 247             if expected_md5:
 248                 file_pkg.seek(0, 0)
 249                 file_md5 = md5()
 250                 chunk_data = file_pkg.read(1024)
 251                 while chunk_data:
 252                     file_md5.update(chunk_data)
 253                     chunk_data = file_pkg.read(1024)
 254                 if expected_md5 != file_md5.hexdigest():
 255                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 256             file_pkg.seek(0, 0)
 257             if compressed == "gzip":
 258                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 259                 descriptor_file_name = None
 260                 for tarinfo in tar:
 261                     tarname = tarinfo.name
 262                     tarname_path = tarname.split("/")
 263                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 264                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 265                     if len(tarname_path) == 1 and not tarinfo.isdir():
 266                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 267                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 268                         storage["pkg-dir"] = tarname_path[0]
 269                         if len(tarname_path) == 2:
 270                             if descriptor_file_name:
 271                                 raise EngineException(
 272                                     "Found more than one descriptor file at package descriptor tar.gz")
 273                             descriptor_file_name = tarname
 274                 if not descriptor_file_name:
 275                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 276                 storage["descriptor"] = descriptor_file_name
 277                 storage["zipfile"] = filename
 278                 self.fs.file_extract(tar, temp_folder)
 279                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 280                     content = descriptor_file.read()
 281             else:
 282                 content = file_pkg.read()
 283                 storage["descriptor"] = descriptor_file_name = filename
 284
 285             if descriptor_file_name.endswith(".json"):
 286                 error_text = "Invalid json format "
 287                 indata = json.load(content)
 288             else:
 289                 error_text = "Invalid yaml format "
 290                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 291
 292             current_desc["_admin"]["storage"] = storage
 293             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 294             current_desc["_admin"]["operationalState"] = "ENABLED"
 295
 296             indata = self._remove_envelop(indata)
 297
 298             # Override descriptor with query string kwargs
 299             if kwargs:
 300                 self._update_input_with_kwargs(indata, kwargs)
 301             # it will call overrides method at VnfdTopic or NsdTopic
 302             # indata = self._validate_input_edit(indata, force=session["force"])
 303
 304             deep_update_rfc7396(current_desc, indata)
 305             self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 306             current_desc["_admin"]["modified"] = time()
 307             self.db.replace(self.topic, _id, current_desc)
 308             self.fs.dir_rename(temp_folder, _id)
 309
 310             indata["_id"] = _id
 311             self._send_msg("edited", indata)
 312
 313             # TODO if descriptor has changed because kwargs update content and remove cached zip
 314             # TODO if zip is not present creates one
 315             return True
 316
 317         except EngineException:
 318             raise
 319         except IndexError:
 320             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 321                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 322         except IOError as e:
 323             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 324         except tarfile.ReadError as e:
 325             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 326         except (ValueError, yaml.YAMLError) as e:
 327             raise EngineException(error_text + str(e))
 328         except ValidationError as e:
 329             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 330         finally:
 331             if file_pkg:
 332                 file_pkg.close()
 333
 334     def get_file(self, session, _id, path=None, accept_header=None):
 335         """
 336         Return the file content of a vnfd or nsd
 337         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 338         :param _id: Identity of the vnfd, nsd
 339         :param path: artifact path or "$DESCRIPTOR" or None
 340         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 341         :return: opened file plus Accept format or raises an exception
 342         """
 343         accept_text = accept_zip = False
 344         if accept_header:
 345             if 'text/plain' in accept_header or '*/*' in accept_header:
 346                 accept_text = True
 347             if 'application/zip' in accept_header or '*/*' in accept_header:
 348                 accept_zip = 'application/zip'
 349             elif 'application/gzip' in accept_header:
 350                 accept_zip = 'application/gzip'
 351
 352         if not accept_text and not accept_zip:
 353             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 354                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 355
 356         content = self.show(session, _id)
 357         if content["_admin"]["onboardingState"] != "ONBOARDED":
 358             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 359                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 360                                   http_code=HTTPStatus.CONFLICT)
 361         storage = content["_admin"]["storage"]
 362         if path is not None and path != "$DESCRIPTOR":   # artifacts
 363             if not storage.get('pkg-dir'):
 364                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 365             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 366                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 367                 return folder_content, "text/plain"
 368                 # TODO manage folders in http
 369             else:
 370                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 371                     "application/octet-stream"
 372
 373         # pkgtype   accept  ZIP  TEXT    -> result
 374         # manyfiles         yes  X       -> zip
 375         #                   no   yes     -> error
 376         # onefile           yes  no      -> zip
 377         #                   X    yes     -> text
 378
 379         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 380             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 381         elif storage.get('pkg-dir') and not accept_zip:
 382             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 383                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 384         else:
 385             if not storage.get('zipfile'):
 386                 # TODO generate zipfile if not present
 387                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 388                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 389             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 390
 391     def pyangbind_validation(self, item, data, force=False):
 392         try:
 393             if item == "vnfds":
 394                 myvnfd = vnfd_im()
 395                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 396                                                  path_helper=True, skip_unknown=force)
 397                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 398             elif item == "nsds":
 399                 mynsd = nsd_im()
 400                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 401                                                  path_helper=True, skip_unknown=force)
 402                 out = pybindJSON.dumps(mynsd, mode="ietf")
 403             elif item == "nsts":
 404                 mynst = nst_im()
 405                 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
 406                                                  path_helper=True, skip_unknown=force)
 407                 out = pybindJSON.dumps(mynst, mode="ietf")
 408             else:
 409                 raise EngineException("Not possible to validate '{}' item".format(item),
 410                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 411
 412             desc_out = self._remove_envelop(yaml.safe_load(out))
 413             return desc_out
 414
 415         except Exception as e:
 416             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 417                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 418
 419
 420 class VnfdTopic(DescriptorTopic):
 421     topic = "vnfds"
 422     topic_msg = "vnfd"
 423
 424     def __init__(self, db, fs, msg, auth):
 425         DescriptorTopic.__init__(self, db, fs, msg, auth)
 426
 427     @staticmethod
 428     def _remove_envelop(indata=None):
 429         if not indata:
 430             return {}
 431         clean_indata = indata
 432         if clean_indata.get('vnfd:vnfd-catalog'):
 433             clean_indata = clean_indata['vnfd:vnfd-catalog']
 434         elif clean_indata.get('vnfd-catalog'):
 435             clean_indata = clean_indata['vnfd-catalog']
 436         if clean_indata.get('vnfd'):
 437             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 438                 raise EngineException("'vnfd' must be a list of only one element")
 439             clean_indata = clean_indata['vnfd'][0]
 440         elif clean_indata.get('vnfd:vnfd'):
 441             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 442                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 443             clean_indata = clean_indata['vnfd:vnfd'][0]
 444         return clean_indata
 445
 446     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 447         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 448
 449         # set type of vnfd
 450         contains_pdu = False
 451         contains_vdu = False
 452         for vdu in get_iterable(final_content.get("vdu")):
 453             if vdu.get("pdu-type"):
 454                 contains_pdu = True
 455             else:
 456                 contains_vdu = True
 457         if contains_pdu:
 458             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 459         elif contains_vdu:
 460             final_content["_admin"]["type"] = "vnfd"
 461         # if neither vud nor pdu do not fill type
 462
 463     def check_conflict_on_del(self, session, _id, db_content):
 464         """
 465         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 466         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 467         that uses this vnfd
 468         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 469         :param _id: vnfd internal id
 470         :param db_content: The database content of the _id.
 471         :return: None or raises EngineException with the conflict
 472         """
 473         if session["force"]:
 474             return
 475         descriptor = db_content
 476         descriptor_id = descriptor.get("id")
 477         if not descriptor_id:  # empty vnfd not uploaded
 478             return
 479
 480         _filter = self._get_project_filter(session)
 481
 482         # check vnfrs using this vnfd
 483         _filter["vnfd-id"] = _id
 484         if self.db.get_list("vnfrs", _filter):
 485             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 486
 487         # check NSD referencing this VNFD
 488         del _filter["vnfd-id"]
 489         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 490         if self.db.get_list("nsds", _filter):
 491             raise EngineException("There is at least one NSD referencing this descriptor",
 492                                   http_code=HTTPStatus.CONFLICT)
 493
 494     def _validate_input_new(self, indata, storage_params, force=False):
 495         indata = self.pyangbind_validation("vnfds", indata, force)
 496         # Cross references validation in the descriptor
 497         if indata.get("vdu"):
 498             if not indata.get("mgmt-interface"):
 499                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 500                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 501             if indata["mgmt-interface"].get("cp"):
 502                 for cp in get_iterable(indata.get("connection-point")):
 503                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 504                         break
 505                 else:
 506                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 507                                           .format(indata["mgmt-interface"]["cp"]),
 508                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 509
 510         for vdu in get_iterable(indata.get("vdu")):
 511             for interface in get_iterable(vdu.get("interface")):
 512                 if interface.get("external-connection-point-ref"):
 513                     for cp in get_iterable(indata.get("connection-point")):
 514                         if cp["name"] == interface["external-connection-point-ref"]:
 515                             break
 516                     else:
 517                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 518                                               "must match an existing connection-point"
 519                                               .format(vdu["id"], interface["name"],
 520                                                       interface["external-connection-point-ref"]),
 521                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 522
 523                 elif interface.get("internal-connection-point-ref"):
 524                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 525                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 526                             break
 527                     else:
 528                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 529                                               "must match an existing vdu:internal-connection-point"
 530                                               .format(vdu["id"], interface["name"],
 531                                                       interface["internal-connection-point-ref"]),
 532                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 533             # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 534             if vdu.get("vdu-configuration"):
 535                 if vdu["vdu-configuration"].get("juju"):
 536                     if not self._validate_package_folders(storage_params, 'charms'):
 537                         raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
 538                                               "package".format(indata["id"], vdu["id"]))
 539             # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
 540             if vdu.get("cloud-init-file"):
 541                 if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 542                     raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 543                                           "package".format(indata["id"], vdu["id"]))
 544         # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 545         if indata.get("vnf-configuration"):
 546             if indata["vnf-configuration"].get("juju"):
 547                 if not self._validate_package_folders(storage_params, 'charms'):
 548                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 549                                           "package".format(indata["id"]))
 550         vld_names = []  # For detection of duplicated VLD names
 551         for ivld in get_iterable(indata.get("internal-vld")):
 552             # BEGIN Detection of duplicated VLD names
 553             ivld_name = ivld["name"]
 554             if ivld_name in vld_names:
 555                 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
 556                                       .format(ivld["name"], indata["id"], ivld["id"]),
 557                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 558             else:
 559                 vld_names.append(ivld_name)
 560             # END Detection of duplicated VLD names
 561             for icp in get_iterable(ivld.get("internal-connection-point")):
 562                 icp_mark = False
 563                 for vdu in get_iterable(indata.get("vdu")):
 564                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 565                         if icp["id-ref"] == internal_cp["id"]:
 566                             icp_mark = True
 567                             break
 568                     if icp_mark:
 569                         break
 570                 else:
 571                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 572                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 573                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 574             if ivld.get("ip-profile-ref"):
 575                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 576                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 577                         break
 578                 else:
 579                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 580                         ivld["id"], ivld["ip-profile-ref"]),
 581                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 582         for mp in get_iterable(indata.get("monitoring-param")):
 583             if mp.get("vdu-monitoring-param"):
 584                 mp_vmp_mark = False
 585                 for vdu in get_iterable(indata.get("vdu")):
 586                     for vmp in get_iterable(vdu.get("monitoring-param")):
 587                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 588                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 589                             mp_vmp_mark = True
 590                             break
 591                     if mp_vmp_mark:
 592                         break
 593                 else:
 594                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 595                                           "defined at vdu[id='{}'] or vdu does not exist"
 596                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 597                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 598                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 599             elif mp.get("vdu-metric"):
 600                 mp_vm_mark = False
 601                 for vdu in get_iterable(indata.get("vdu")):
 602                     if vdu.get("vdu-configuration"):
 603                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 604                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 605                                     mp["vdu-metric"]["vdu-ref"]:
 606                                 mp_vm_mark = True
 607                                 break
 608                         if mp_vm_mark:
 609                             break
 610                 else:
 611                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 612                                           "vdu[id='{}'] or vdu does not exist"
 613                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 614                                                   mp["vdu-metric"]["vdu-ref"]),
 615                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 616
 617         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 618             for sp in get_iterable(sgd.get("scaling-policy")):
 619                 for sc in get_iterable(sp.get("scaling-criteria")):
 620                     for mp in get_iterable(indata.get("monitoring-param")):
 621                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 622                             break
 623                     else:
 624                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 625                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 626                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 627                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 628             for sgd_vdu in get_iterable(sgd.get("vdu")):
 629                 sgd_vdu_mark = False
 630                 for vdu in get_iterable(indata.get("vdu")):
 631                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 632                         sgd_vdu_mark = True
 633                         break
 634                 if sgd_vdu_mark:
 635                     break
 636             else:
 637                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 638                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 639                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 640             for sca in get_iterable(sgd.get("scaling-config-action")):
 641                 if not indata.get("vnf-configuration"):
 642                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 643                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 644                                           .format(sgd["name"]),
 645                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 646                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 647                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 648                         break
 649                 else:
 650                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 651                                           "primitive-name-ref='{}' does not match any "
 652                                           "vnf-configuration:config-primitive:name"
 653                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 654                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 655         return indata
 656
 657     def _validate_input_edit(self, indata, force=False):
 658         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 659         return indata
 660
 661     def _validate_package_folders(self, storage_params, folder, file=None):
 662         if not storage_params or not storage_params.get("pkg-dir"):
 663             return False
 664         else:
 665             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 666                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 667             else:
 668                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 669             if file:
 670                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 671             else:
 672                 if self.fs.file_exists(f, 'dir'):
 673                     if self.fs.dir_ls(f):
 674                         return True
 675             return False
 676
 677
 678 class NsdTopic(DescriptorTopic):
 679     topic = "nsds"
 680     topic_msg = "nsd"
 681
 682     def __init__(self, db, fs, msg, auth):
 683         DescriptorTopic.__init__(self, db, fs, msg, auth)
 684
 685     @staticmethod
 686     def _remove_envelop(indata=None):
 687         if not indata:
 688             return {}
 689         clean_indata = indata
 690
 691         if clean_indata.get('nsd:nsd-catalog'):
 692             clean_indata = clean_indata['nsd:nsd-catalog']
 693         elif clean_indata.get('nsd-catalog'):
 694             clean_indata = clean_indata['nsd-catalog']
 695         if clean_indata.get('nsd'):
 696             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 697                 raise EngineException("'nsd' must be a list of only one element")
 698             clean_indata = clean_indata['nsd'][0]
 699         elif clean_indata.get('nsd:nsd'):
 700             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 701                 raise EngineException("'nsd:nsd' must be a list of only one element")
 702             clean_indata = clean_indata['nsd:nsd'][0]
 703         return clean_indata
 704
 705     def _validate_input_new(self, indata, storage_params, force=False):
 706         indata = self.pyangbind_validation("nsds", indata, force)
 707         # Cross references validation in the descriptor
 708         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 709         for vld in get_iterable(indata.get("vld")):
 710             if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
 711                 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
 712                                       " You cannot set an ip-profile when mgmt-network is True"
 713                                       .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 714             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 715                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 716                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 717                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 718                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 719                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 720                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 721                                                                  constituent_vnfd["member-vnf-index"],
 722                                                                  constituent_vnfd["vnfd-id-ref"]),
 723                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 724                         break
 725                 else:
 726                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 727                                           "does not match any constituent-vnfd:member-vnf-index"
 728                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 729                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 730         # Check VNFFGD
 731         for fgd in get_iterable(indata.get("vnffgd")):
 732             for cls in get_iterable(fgd.get("classifier")):
 733                 rspref = cls.get("rsp-id-ref")
 734                 for rsp in get_iterable(fgd.get("rsp")):
 735                     rspid = rsp.get("id")
 736                     if rspid and rspref and rspid == rspref:
 737                         break
 738                 else:
 739                     raise EngineException(
 740                         "Error at vnffgd[id='{}']:classifier[id='{}']:rsp-id-ref '{}' does not match any rsp:id"
 741                         .format(fgd["id"], cls["id"], rspref),
 742                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 743         return indata
 744
 745     def _validate_input_edit(self, indata, force=False):
 746         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 747         return indata
 748
 749     def _check_descriptor_dependencies(self, session, descriptor):
 750         """
 751         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 752         connection points are ok
 753         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 754         :param descriptor: descriptor to be inserted or edit
 755         :return: None or raises exception
 756         """
 757         if session["force"]:
 758             return
 759         member_vnfd_index = {}
 760         if descriptor.get("constituent-vnfd") and not session["force"]:
 761             for vnf in descriptor["constituent-vnfd"]:
 762                 vnfd_id = vnf["vnfd-id-ref"]
 763                 filter_q = self._get_project_filter(session)
 764                 filter_q["id"] = vnfd_id
 765                 vnf_list = self.db.get_list("vnfds", filter_q)
 766                 if not vnf_list:
 767                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 768                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 769                 # elif len(vnf_list) > 1:
 770                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 771                 #                           http_code=HTTPStatus.CONFLICT)
 772                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 773
 774         # Cross references validation in the descriptor and vnfd connection point validation
 775         for vld in get_iterable(descriptor.get("vld")):
 776             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 777                 # look if this vnfd contains this connection point
 778                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 779                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 780                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 781                         break
 782                 else:
 783                     raise EngineException(
 784                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 785                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 786                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 787                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 788                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 789
 790     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 791         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 792
 793         self._check_descriptor_dependencies(session, final_content)
 794
 795     def check_conflict_on_del(self, session, _id, db_content):
 796         """
 797         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 798         that NSD can be public and be used by other projects.
 799         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 800         :param _id: nsd internal id
 801         :param db_content: The database content of the _id
 802         :return: None or raises EngineException with the conflict
 803         """
 804         if session["force"]:
 805             return
 806         descriptor = db_content
 807         descriptor_id = descriptor.get("id")
 808         if not descriptor_id:  # empty nsd not uploaded
 809             return
 810
 811         # check NSD used by NS
 812         _filter = self._get_project_filter(session)
 813         _filter["nsd-id"] = _id
 814         if self.db.get_list("nsrs", _filter):
 815             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
 816
 817         # check NSD referenced by NST
 818         del _filter["nsd-id"]
 819         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
 820         if self.db.get_list("nsts", _filter):
 821             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
 822                                   http_code=HTTPStatus.CONFLICT)
 823
 824
 825 class NstTopic(DescriptorTopic):
 826     topic = "nsts"
 827     topic_msg = "nst"
 828
 829     def __init__(self, db, fs, msg, auth):
 830         DescriptorTopic.__init__(self, db, fs, msg, auth)
 831
 832     @staticmethod
 833     def _remove_envelop(indata=None):
 834         if not indata:
 835             return {}
 836         clean_indata = indata
 837
 838         if clean_indata.get('nst'):
 839             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 840                 raise EngineException("'nst' must be a list only one element")
 841             clean_indata = clean_indata['nst'][0]
 842         elif clean_indata.get('nst:nst'):
 843             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
 844                 raise EngineException("'nst:nst' must be a list only one element")
 845             clean_indata = clean_indata['nst:nst'][0]
 846         return clean_indata
 847
 848     def _validate_input_edit(self, indata, force=False):
 849         # TODO validate with pyangbind, serialize
 850         return indata
 851
 852     def _validate_input_new(self, indata, storage_params, force=False):
 853         indata = self.pyangbind_validation("nsts", indata, force)
 854         return indata.copy()
 855
 856     def _check_descriptor_dependencies(self, session, descriptor):
 857         """
 858         Check that the dependent descriptors exist on a new descriptor or edition
 859         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 860         :param descriptor: descriptor to be inserted or edit
 861         :return: None or raises exception
 862         """
 863         if not descriptor.get("netslice-subnet"):
 864             return
 865         for nsd in descriptor["netslice-subnet"]:
 866             nsd_id = nsd["nsd-ref"]
 867             filter_q = self._get_project_filter(session)
 868             filter_q["id"] = nsd_id
 869             if not self.db.get_list("nsds", filter_q):
 870                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
 871                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
 872
 873     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 874         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 875
 876         self._check_descriptor_dependencies(session, final_content)
 877
 878     def check_conflict_on_del(self, session, _id, db_content):
 879         """
 880         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
 881         that NST can be public and be used by other projects.
 882         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 883         :param _id: nst internal id
 884         :param db_content: The database content of the _id.
 885         :return: None or raises EngineException with the conflict
 886         """
 887         # TODO: Check this method
 888         if session["force"]:
 889             return
 890         # Get Network Slice Template from Database
 891         _filter = self._get_project_filter(session)
 892         _filter["_admin.nst-id"] = _id
 893         if self.db.get_list("nsis", _filter):
 894             raise EngineException("there is at least one Netslice Instance using this descriptor",
 895                                   http_code=HTTPStatus.CONFLICT)
 896
 897
 898 class PduTopic(BaseTopic):
 899     topic = "pdus"
 900     topic_msg = "pdu"
 901     schema_new = pdu_new_schema
 902     schema_edit = pdu_edit_schema
 903
 904     def __init__(self, db, fs, msg, auth):
 905         BaseTopic.__init__(self, db, fs, msg, auth)
 906
 907     @staticmethod
 908     def format_on_new(content, project_id=None, make_public=False):
 909         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 910         content["_admin"]["onboardingState"] = "CREATED"
 911         content["_admin"]["operationalState"] = "ENABLED"
 912         content["_admin"]["usageState"] = "NOT_IN_USE"
 913
 914     def check_conflict_on_del(self, session, _id, db_content):
 915         """
 916         Check that there is not any vnfr that uses this PDU
 917         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 918         :param _id: pdu internal id
 919         :param db_content: The database content of the _id.
 920         :return: None or raises EngineException with the conflict
 921         """
 922         if session["force"]:
 923             return
 924
 925         _filter = self._get_project_filter(session)
 926         _filter["vdur.pdu-id"] = _id
 927         if self.db.get_list("vnfrs", _filter):
 928             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)