osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from time import time
  24 from uuid import uuid4
  25 from re import fullmatch
  26 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema, \
  27     validate_input, vnfpkgop_new_schema
  28 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  29 from osm_im.vnfd import vnfd as vnfd_im
  30 from osm_im.nsd import nsd as nsd_im
  31 from osm_im.nst import nst as nst_im
  32 from pyangbind.lib.serialise import pybindJSONDecoder
  33 import pyangbind.lib.pybindJSON as pybindJSON
  34
  35 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  36
  37
  38 class DescriptorTopic(BaseTopic):
  39
  40     def __init__(self, db, fs, msg, auth):
  41         BaseTopic.__init__(self, db, fs, msg, auth)
  42
  43     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  44         super().check_conflict_on_edit(session, final_content, edit_content, _id)
  45
  46         def _check_unique_id_name(descriptor, position=""):
  47             for desc_key, desc_item in descriptor.items():
  48                 if isinstance(desc_item, list) and desc_item:
  49                     used_ids = []
  50                     desc_item_id = None
  51                     for index, list_item in enumerate(desc_item):
  52                         if isinstance(list_item, dict):
  53                             _check_unique_id_name(list_item, "{}.{}[{}]"
  54                                                   .format(position, desc_key, index))
  55                             # Base case
  56                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  57                                 desc_item_id = "id" if list_item.get("id") else "name"
  58                             if desc_item_id and list_item.get(desc_item_id):
  59                                 if list_item[desc_item_id] in used_ids:
  60                                     position = "{}.{}[{}]".format(position, desc_key, index)
  61                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  62                                                           .format(desc_item_id, list_item[desc_item_id],
  63                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  64                                 used_ids.append(list_item[desc_item_id])
  65
  66         _check_unique_id_name(final_content)
  67         # 1. validate again with pyangbind
  68         # 1.1. remove internal keys
  69         internal_keys = {}
  70         for k in ("_id", "_admin"):
  71             if k in final_content:
  72                 internal_keys[k] = final_content.pop(k)
  73         storage_params = internal_keys["_admin"].get("storage")
  74         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  75         # 1.2. modify final_content with a serialized version
  76         final_content.clear()
  77         final_content.update(serialized)
  78         # 1.3. restore internal keys
  79         for k, v in internal_keys.items():
  80             final_content[k] = v
  81
  82         if session["force"]:
  83             return
  84         # 2. check that this id is not present
  85         if "id" in edit_content:
  86             _filter = self._get_project_filter(session)
  87             _filter["id"] = final_content["id"]
  88             _filter["_id.neq"] = _id
  89             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  90                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  91                                                                                                final_content["id"]),
  92                                       HTTPStatus.CONFLICT)
  93
  94     @staticmethod
  95     def format_on_new(content, project_id=None, make_public=False):
  96         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  97         content["_admin"]["onboardingState"] = "CREATED"
  98         content["_admin"]["operationalState"] = "DISABLED"
  99         content["_admin"]["usageState"] = "NOT_IN_USE"
 100
 101     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 102         """
 103         Deletes file system storage associated with the descriptor
 104         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 105         :param _id: server internal id
 106         :param db_content: The database content of the descriptor
 107         :param not_send_msg: To not send message (False) or store content (list) instead
 108         :return: None if ok or raises EngineException with the problem
 109         """
 110         self.fs.file_delete(_id, ignore_non_exist=True)
 111         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 112
 113     @staticmethod
 114     def get_one_by_id(db, session, topic, id):
 115         # find owned by this project
 116         _filter = BaseTopic._get_project_filter(session)
 117         _filter["id"] = id
 118         desc_list = db.get_list(topic, _filter)
 119         if len(desc_list) == 1:
 120             return desc_list[0]
 121         elif len(desc_list) > 1:
 122             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 123                               HTTPStatus.CONFLICT)
 124
 125         # not found any: try to find public
 126         _filter = BaseTopic._get_project_filter(session)
 127         _filter["id"] = id
 128         desc_list = db.get_list(topic, _filter)
 129         if not desc_list:
 130             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 131         elif len(desc_list) == 1:
 132             return desc_list[0]
 133         else:
 134             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 135                 topic[:-1], id), HTTPStatus.CONFLICT)
 136
 137     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 138         """
 139         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 140         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 141         (self.upload_content)
 142         :param rollback: list to append created items at database in case a rollback may to be done
 143         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 144         :param indata: data to be inserted
 145         :param kwargs: used to override the indata descriptor
 146         :param headers: http request headers
 147         :return: _id, None: identity of the inserted data; and None as there is not any operation
 148         """
 149
 150         # No needed to capture exceptions
 151         # Check Quota
 152         self.check_quota(session)
 153
 154         # _remove_envelop
 155         if indata:
 156             if "userDefinedData" in indata:
 157                 indata = indata['userDefinedData']
 158
 159         # Override descriptor with query string kwargs
 160         self._update_input_with_kwargs(indata, kwargs)
 161         # uncomment when this method is implemented.
 162         # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 163         # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 164
 165         content = {"_admin": {"userDefinedData": indata}}
 166         self.format_on_new(content, session["project_id"], make_public=session["public"])
 167         _id = self.db.create(self.topic, content)
 168         rollback.append({"topic": self.topic, "_id": _id})
 169         self._send_msg("created", {"_id": _id})
 170         return _id, None
 171
 172     def upload_content(self, session, _id, indata, kwargs, headers):
 173         """
 174         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 175         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 176         :param _id : the nsd,vnfd is already created, this is the id
 177         :param indata: http body request
 178         :param kwargs: user query string to override parameters. NOT USED
 179         :param headers:  http request headers
 180         :return: True if package is completely uploaded or False if partial content has been uploded
 181             Raise exception on error
 182         """
 183         # Check that _id exists and it is valid
 184         current_desc = self.show(session, _id)
 185
 186         content_range_text = headers.get("Content-Range")
 187         expected_md5 = headers.get("Content-File-MD5")
 188         compressed = None
 189         content_type = headers.get("Content-Type")
 190         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 191                 "application/zip" in content_type:
 192             compressed = "gzip"
 193         filename = headers.get("Content-Filename")
 194         if not filename:
 195             filename = "package.tar.gz" if compressed else "package"
 196         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 197         file_pkg = None
 198         error_text = ""
 199         try:
 200             if content_range_text:
 201                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 202                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 203                     raise IndexError()
 204                 start = int(content_range[1])
 205                 end = int(content_range[2]) + 1
 206                 total = int(content_range[3])
 207             else:
 208                 start = 0
 209             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 210
 211             if start:
 212                 if not self.fs.file_exists(temp_folder, 'dir'):
 213                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 214             else:
 215                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 216                 self.fs.mkdir(temp_folder)
 217
 218             storage = self.fs.get_params()
 219             storage["folder"] = _id
 220
 221             file_path = (temp_folder, filename)
 222             if self.fs.file_exists(file_path, 'file'):
 223                 file_size = self.fs.file_size(file_path)
 224             else:
 225                 file_size = 0
 226             if file_size != start:
 227                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 228                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 229             file_pkg = self.fs.file_open(file_path, 'a+b')
 230             if isinstance(indata, dict):
 231                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 232                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 233             else:
 234                 indata_len = 0
 235                 while True:
 236                     indata_text = indata.read(4096)
 237                     indata_len += len(indata_text)
 238                     if not indata_text:
 239                         break
 240                     file_pkg.write(indata_text)
 241             if content_range_text:
 242                 if indata_len != end - start:
 243                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 244                         start, end - 1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 245                 if end != total:
 246                     # TODO update to UPLOADING
 247                     return False
 248
 249             # PACKAGE UPLOADED
 250             if expected_md5:
 251                 file_pkg.seek(0, 0)
 252                 file_md5 = md5()
 253                 chunk_data = file_pkg.read(1024)
 254                 while chunk_data:
 255                     file_md5.update(chunk_data)
 256                     chunk_data = file_pkg.read(1024)
 257                 if expected_md5 != file_md5.hexdigest():
 258                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 259             file_pkg.seek(0, 0)
 260             if compressed == "gzip":
 261                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 262                 descriptor_file_name = None
 263                 for tarinfo in tar:
 264                     tarname = tarinfo.name
 265                     tarname_path = tarname.split("/")
 266                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 267                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 268                     if len(tarname_path) == 1 and not tarinfo.isdir():
 269                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 270                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 271                         storage["pkg-dir"] = tarname_path[0]
 272                         if len(tarname_path) == 2:
 273                             if descriptor_file_name:
 274                                 raise EngineException(
 275                                     "Found more than one descriptor file at package descriptor tar.gz")
 276                             descriptor_file_name = tarname
 277                 if not descriptor_file_name:
 278                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 279                 storage["descriptor"] = descriptor_file_name
 280                 storage["zipfile"] = filename
 281                 self.fs.file_extract(tar, temp_folder)
 282                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 283                     content = descriptor_file.read()
 284             else:
 285                 content = file_pkg.read()
 286                 storage["descriptor"] = descriptor_file_name = filename
 287
 288             if descriptor_file_name.endswith(".json"):
 289                 error_text = "Invalid json format "
 290                 indata = json.load(content)
 291             else:
 292                 error_text = "Invalid yaml format "
 293                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 294
 295             current_desc["_admin"]["storage"] = storage
 296             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 297             current_desc["_admin"]["operationalState"] = "ENABLED"
 298
 299             indata = self._remove_envelop(indata)
 300
 301             # Override descriptor with query string kwargs
 302             if kwargs:
 303                 self._update_input_with_kwargs(indata, kwargs)
 304             # it will call overrides method at VnfdTopic or NsdTopic
 305             # indata = self._validate_input_edit(indata, force=session["force"])
 306
 307             deep_update_rfc7396(current_desc, indata)
 308             self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 309             current_desc["_admin"]["modified"] = time()
 310             self.db.replace(self.topic, _id, current_desc)
 311             self.fs.dir_rename(temp_folder, _id)
 312
 313             indata["_id"] = _id
 314             self._send_msg("edited", indata)
 315
 316             # TODO if descriptor has changed because kwargs update content and remove cached zip
 317             # TODO if zip is not present creates one
 318             return True
 319
 320         except EngineException:
 321             raise
 322         except IndexError:
 323             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 324                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 325         except IOError as e:
 326             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 327         except tarfile.ReadError as e:
 328             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 329         except (ValueError, yaml.YAMLError) as e:
 330             raise EngineException(error_text + str(e))
 331         except ValidationError as e:
 332             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 333         finally:
 334             if file_pkg:
 335                 file_pkg.close()
 336
 337     def get_file(self, session, _id, path=None, accept_header=None):
 338         """
 339         Return the file content of a vnfd or nsd
 340         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 341         :param _id: Identity of the vnfd, nsd
 342         :param path: artifact path or "$DESCRIPTOR" or None
 343         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 344         :return: opened file plus Accept format or raises an exception
 345         """
 346         accept_text = accept_zip = False
 347         if accept_header:
 348             if 'text/plain' in accept_header or '*/*' in accept_header:
 349                 accept_text = True
 350             if 'application/zip' in accept_header or '*/*' in accept_header:
 351                 accept_zip = 'application/zip'
 352             elif 'application/gzip' in accept_header:
 353                 accept_zip = 'application/gzip'
 354
 355         if not accept_text and not accept_zip:
 356             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 357                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 358
 359         content = self.show(session, _id)
 360         if content["_admin"]["onboardingState"] != "ONBOARDED":
 361             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 362                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 363                                   http_code=HTTPStatus.CONFLICT)
 364         storage = content["_admin"]["storage"]
 365         if path is not None and path != "$DESCRIPTOR":  # artifacts
 366             if not storage.get('pkg-dir'):
 367                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 368             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 369                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 370                 return folder_content, "text/plain"
 371                 # TODO manage folders in http
 372             else:
 373                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \
 374                     "application/octet-stream"
 375
 376         # pkgtype   accept  ZIP  TEXT    -> result
 377         # manyfiles         yes  X       -> zip
 378         #                   no   yes     -> error
 379         # onefile           yes  no      -> zip
 380         #                   X    yes     -> text
 381         contain_many_files = False
 382         if storage.get('pkg-dir'):
 383             # check if there are more than one file in the package, ignoring checksums.txt.
 384             pkg_files = self.fs.dir_ls((storage['folder'], storage['pkg-dir']))
 385             if len(pkg_files) >= 3 or (len(pkg_files) == 2 and 'checksums.txt' not in pkg_files):
 386                 contain_many_files = True
 387         if accept_text and (not contain_many_files or path == "$DESCRIPTOR"):
 388             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 389         elif contain_many_files and not accept_zip:
 390             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 391                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 392         else:
 393             if not storage.get('zipfile'):
 394                 # TODO generate zipfile if not present
 395                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 396                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 397             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 398
 399     def pyangbind_validation(self, item, data, force=False):
 400         try:
 401             if item == "vnfds":
 402                 myvnfd = vnfd_im()
 403                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 404                                                  path_helper=True, skip_unknown=force)
 405                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 406             elif item == "nsds":
 407                 mynsd = nsd_im()
 408                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 409                                                  path_helper=True, skip_unknown=force)
 410                 out = pybindJSON.dumps(mynsd, mode="ietf")
 411             elif item == "nsts":
 412                 mynst = nst_im()
 413                 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
 414                                                  path_helper=True, skip_unknown=force)
 415                 out = pybindJSON.dumps(mynst, mode="ietf")
 416             else:
 417                 raise EngineException("Not possible to validate '{}' item".format(item),
 418                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 419
 420             desc_out = self._remove_envelop(yaml.safe_load(out))
 421             return desc_out
 422
 423         except Exception as e:
 424             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 425                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 426
 427     def _validate_input_edit(self, indata, content, force=False):
 428         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 429         if "_id" in indata:
 430             indata.pop("_id")
 431         if "_admin" not in indata:
 432             indata["_admin"] = {}
 433
 434         if "operationalState" in indata:
 435             if indata["operationalState"] in ("ENABLED", "DISABLED"):
 436                 indata["_admin"]["operationalState"] = indata.pop("operationalState")
 437             else:
 438                 raise EngineException("State '{}' is not a valid operational state"
 439                                       .format(indata["operationalState"]),
 440                                       http_code=HTTPStatus.BAD_REQUEST)
 441
 442         # In the case of user defined data, we need to put the data in the root of the object
 443         # to preserve current expected behaviour
 444         if "userDefinedData" in indata:
 445             data = indata.pop("userDefinedData")
 446             if type(data) == dict:
 447                 indata["_admin"]["userDefinedData"] = data
 448             else:
 449                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 450                                       .format(type(data)),
 451                                       http_code=HTTPStatus.BAD_REQUEST)
 452
 453         if ("operationalState" in indata["_admin"] and
 454                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 455             raise EngineException("operationalState already {}".format(content["_admin"]["operationalState"]),
 456                                   http_code=HTTPStatus.CONFLICT)
 457
 458         return indata
 459
 460
 461 class VnfdTopic(DescriptorTopic):
 462     topic = "vnfds"
 463     topic_msg = "vnfd"
 464
 465     def __init__(self, db, fs, msg, auth):
 466         DescriptorTopic.__init__(self, db, fs, msg, auth)
 467
 468     @staticmethod
 469     def _remove_envelop(indata=None):
 470         if not indata:
 471             return {}
 472         clean_indata = indata
 473         if clean_indata.get('vnfd:vnfd-catalog'):
 474             clean_indata = clean_indata['vnfd:vnfd-catalog']
 475         elif clean_indata.get('vnfd-catalog'):
 476             clean_indata = clean_indata['vnfd-catalog']
 477         if clean_indata.get('vnfd'):
 478             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 479                 raise EngineException("'vnfd' must be a list of only one element")
 480             clean_indata = clean_indata['vnfd'][0]
 481         elif clean_indata.get('vnfd:vnfd'):
 482             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 483                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 484             clean_indata = clean_indata['vnfd:vnfd'][0]
 485         return clean_indata
 486
 487     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 488         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 489
 490         # set type of vnfd
 491         contains_pdu = False
 492         contains_vdu = False
 493         for vdu in get_iterable(final_content.get("vdu")):
 494             if vdu.get("pdu-type"):
 495                 contains_pdu = True
 496             else:
 497                 contains_vdu = True
 498         if contains_pdu:
 499             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 500         elif contains_vdu:
 501             final_content["_admin"]["type"] = "vnfd"
 502         # if neither vud nor pdu do not fill type
 503
 504     def check_conflict_on_del(self, session, _id, db_content):
 505         """
 506         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 507         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 508         that uses this vnfd
 509         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 510         :param _id: vnfd internal id
 511         :param db_content: The database content of the _id.
 512         :return: None or raises EngineException with the conflict
 513         """
 514         if session["force"]:
 515             return
 516         descriptor = db_content
 517         descriptor_id = descriptor.get("id")
 518         if not descriptor_id:  # empty vnfd not uploaded
 519             return
 520
 521         _filter = self._get_project_filter(session)
 522
 523         # check vnfrs using this vnfd
 524         _filter["vnfd-id"] = _id
 525         if self.db.get_list("vnfrs", _filter):
 526             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 527
 528         # check NSD referencing this VNFD
 529         del _filter["vnfd-id"]
 530         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 531         if self.db.get_list("nsds", _filter):
 532             raise EngineException("There is at least one NSD referencing this descriptor",
 533                                   http_code=HTTPStatus.CONFLICT)
 534
 535     def _validate_input_new(self, indata, storage_params, force=False):
 536         indata.pop("onboardingState", None)
 537         indata.pop("operationalState", None)
 538         indata.pop("usageState", None)
 539
 540         indata.pop("links", None)
 541
 542         indata = self.pyangbind_validation("vnfds", indata, force)
 543         # Cross references validation in the descriptor
 544         self.validate_mgmt_interfaces_connection_points(indata)
 545
 546         for vdu in get_iterable(indata.get("vdu")):
 547             self.validate_vdu_connection_point_refs(vdu, indata)
 548             self._validate_vdu_charms_in_package(storage_params, vdu, indata)
 549             self._validate_vdu_cloud_init_in_package(storage_params, vdu, indata)
 550
 551         self._validate_vnf_charms_in_package(storage_params, indata)
 552
 553         self.validate_internal_vlds(indata)
 554         self.validate_monitoring_params(indata)
 555         self.validate_scaling_group_descriptor(indata)
 556
 557         return indata
 558
 559     @staticmethod
 560     def validate_mgmt_interfaces_connection_points(indata):
 561         if not indata.get("vdu"):
 562             return
 563         if not indata.get("mgmt-interface"):
 564             raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 565                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 566         if indata["mgmt-interface"].get("cp"):
 567             for cp in get_iterable(indata.get("connection-point")):
 568                 if cp["name"] == indata["mgmt-interface"]["cp"]:
 569                     break
 570             else:
 571                 raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 572                                       .format(indata["mgmt-interface"]["cp"]),
 573                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 574
 575     @staticmethod
 576     def validate_vdu_connection_point_refs(vdu, indata):
 577         icp_refs = []
 578         ecp_refs = []
 579         for interface in get_iterable(vdu.get("interface")):
 580             if interface.get("external-connection-point-ref"):
 581                 if interface.get("external-connection-point-ref") in ecp_refs:
 582                     raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 583                                           "is referenced by other interface"
 584                                           .format(vdu["id"], interface["name"],
 585                                                   interface["external-connection-point-ref"]),
 586                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 587                 ecp_refs.append(interface.get("external-connection-point-ref"))
 588                 for cp in get_iterable(indata.get("connection-point")):
 589                     if cp["name"] == interface["external-connection-point-ref"]:
 590                         break
 591                 else:
 592                     raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 593                                           "must match an existing connection-point"
 594                                           .format(vdu["id"], interface["name"],
 595                                                   interface["external-connection-point-ref"]),
 596                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 597             elif interface.get("internal-connection-point-ref"):
 598                 if interface.get("internal-connection-point-ref") in icp_refs:
 599                     raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 600                                           "is referenced by other interface"
 601                                           .format(vdu["id"], interface["name"],
 602                                                   interface["internal-connection-point-ref"]),
 603                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 604                 icp_refs.append(interface.get("internal-connection-point-ref"))
 605                 for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 606                     if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 607                         break
 608                 else:
 609                     raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 610                                           "must match an existing vdu:internal-connection-point"
 611                                           .format(vdu["id"], interface["name"],
 612                                                   interface["internal-connection-point-ref"]),
 613                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 614
 615     def _validate_vdu_charms_in_package(self, storage_params, vdu, indata):
 616         if not vdu.get("vdu-configuration"):
 617             return
 618         if vdu["vdu-configuration"].get("juju"):
 619             if not self._validate_package_folders(storage_params, 'charms'):
 620                 raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
 621                                       "package".format(indata["id"], vdu["id"]))
 622
 623     def _validate_vdu_cloud_init_in_package(self, storage_params, vdu, indata):
 624         if not vdu.get("cloud-init-file"):
 625             return
 626         if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 627             raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 628                                   "package".format(indata["id"], vdu["id"]))
 629
 630     def _validate_vnf_charms_in_package(self, storage_params, indata):
 631         if not indata.get("vnf-configuration"):
 632             return
 633         if indata["vnf-configuration"].get("juju"):
 634             if not self._validate_package_folders(storage_params, 'charms'):
 635                 raise EngineException("Charm defined in vnf[id={}] but not present in "
 636                                       "package".format(indata["id"]))
 637
 638     def _validate_package_folders(self, storage_params, folder, file=None):
 639         if not storage_params or not storage_params.get("pkg-dir"):
 640             return False
 641         else:
 642             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 643                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 644             else:
 645                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 646             if file:
 647                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 648             else:
 649                 if self.fs.file_exists(f, 'dir'):
 650                     if self.fs.dir_ls(f):
 651                         return True
 652             return False
 653
 654     @staticmethod
 655     def validate_internal_vlds(indata):
 656         vld_names = []  # For detection of duplicated VLD names
 657         for ivld in get_iterable(indata.get("internal-vld")):
 658             ivld_name = ivld.get("name")
 659             if ivld_name and ivld_name in vld_names:
 660                 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
 661                                       .format(ivld["name"], indata["id"], ivld["id"]),
 662                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 663             else:
 664                 vld_names.append(ivld_name)
 665
 666             for icp in get_iterable(ivld.get("internal-connection-point")):
 667                 icp_mark = False
 668                 for vdu in get_iterable(indata.get("vdu")):
 669                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 670                         if icp["id-ref"] == internal_cp["id"]:
 671                             icp_mark = True
 672                             break
 673                     if icp_mark:
 674                         break
 675                 else:
 676                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 677                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 678                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 679             if ivld.get("ip-profile-ref"):
 680                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 681                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 682                         break
 683                 else:
 684                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 685                         ivld["id"], ivld["ip-profile-ref"]),
 686                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 687
 688     @staticmethod
 689     def validate_monitoring_params(indata):
 690         for mp in get_iterable(indata.get("monitoring-param")):
 691             if mp.get("vdu-monitoring-param"):
 692                 mp_vmp_mark = False
 693                 for vdu in get_iterable(indata.get("vdu")):
 694                     for vmp in get_iterable(vdu.get("monitoring-param")):
 695                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] == \
 696                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 697                             mp_vmp_mark = True
 698                             break
 699                     if mp_vmp_mark:
 700                         break
 701                 else:
 702                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 703                                           "defined at vdu[id='{}'] or vdu does not exist"
 704                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 705                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 706                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 707             elif mp.get("vdu-metric"):
 708                 mp_vm_mark = False
 709                 for vdu in get_iterable(indata.get("vdu")):
 710                     if vdu.get("vdu-configuration"):
 711                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 712                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 713                                     mp["vdu-metric"]["vdu-ref"]:
 714                                 mp_vm_mark = True
 715                                 break
 716                         if mp_vm_mark:
 717                             break
 718                 else:
 719                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 720                                           "vdu[id='{}'] or vdu does not exist"
 721                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 722                                                   mp["vdu-metric"]["vdu-ref"]),
 723                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 724
 725     @staticmethod
 726     def validate_scaling_group_descriptor(indata):
 727         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 728             for sp in get_iterable(sgd.get("scaling-policy")):
 729                 for sc in get_iterable(sp.get("scaling-criteria")):
 730                     for mp in get_iterable(indata.get("monitoring-param")):
 731                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 732                             break
 733                     else:
 734                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 735                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 736                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 737                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 738             for sgd_vdu in get_iterable(sgd.get("vdu")):
 739                 sgd_vdu_mark = False
 740                 for vdu in get_iterable(indata.get("vdu")):
 741                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 742                         sgd_vdu_mark = True
 743                         break
 744                 if sgd_vdu_mark:
 745                     break
 746             else:
 747                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 748                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 749                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 750             for sca in get_iterable(sgd.get("scaling-config-action")):
 751                 if not indata.get("vnf-configuration"):
 752                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 753                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 754                                           .format(sgd["name"]),
 755                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 756                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 757                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 758                         break
 759                 else:
 760                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 761                                           "primitive-name-ref='{}' does not match any "
 762                                           "vnf-configuration:config-primitive:name"
 763                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 764                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 765
 766     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 767         """
 768         Deletes associate file system storage (via super)
 769         Deletes associated vnfpkgops from database.
 770         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 771         :param _id: server internal id
 772         :param db_content: The database content of the descriptor
 773         :return: None
 774         :raises: FsException in case of error while deleting associated storage
 775         """
 776         super().delete_extra(session, _id, db_content, not_send_msg)
 777         self.db.del_list("vnfpkgops", {"vnfPkgId": _id})
 778
 779     def sol005_projection(self, data):
 780         data["onboardingState"] = data["_admin"]["onboardingState"]
 781         data["operationalState"] = data["_admin"]["operationalState"]
 782         data["usageState"] = data["_admin"]["usageState"]
 783
 784         links = {}
 785         links["self"] = {"href": "/vnfpkgm/v1/vnf_packages/{}".format(data["_id"])}
 786         links["vnfd"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(data["_id"])}
 787         links["packageContent"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/package_content".format(data["_id"])}
 788         data["_links"] = links
 789
 790         return super().sol005_projection(data)
 791
 792
 793 class NsdTopic(DescriptorTopic):
 794     topic = "nsds"
 795     topic_msg = "nsd"
 796
 797     def __init__(self, db, fs, msg, auth):
 798         DescriptorTopic.__init__(self, db, fs, msg, auth)
 799
 800     @staticmethod
 801     def _remove_envelop(indata=None):
 802         if not indata:
 803             return {}
 804         clean_indata = indata
 805
 806         if clean_indata.get('nsd:nsd-catalog'):
 807             clean_indata = clean_indata['nsd:nsd-catalog']
 808         elif clean_indata.get('nsd-catalog'):
 809             clean_indata = clean_indata['nsd-catalog']
 810         if clean_indata.get('nsd'):
 811             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 812                 raise EngineException("'nsd' must be a list of only one element")
 813             clean_indata = clean_indata['nsd'][0]
 814         elif clean_indata.get('nsd:nsd'):
 815             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 816                 raise EngineException("'nsd:nsd' must be a list of only one element")
 817             clean_indata = clean_indata['nsd:nsd'][0]
 818         return clean_indata
 819
 820     def _validate_input_new(self, indata, storage_params, force=False):
 821         indata.pop("nsdOnboardingState", None)
 822         indata.pop("nsdOperationalState", None)
 823         indata.pop("nsdUsageState", None)
 824
 825         indata.pop("links", None)
 826
 827         indata = self.pyangbind_validation("nsds", indata, force)
 828         # Cross references validation in the descriptor
 829         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 830         for vld in get_iterable(indata.get("vld")):
 831             self.validate_vld_mgmt_network_with_ip_profile_ref(vld)
 832             self.validate_vld_connection_point_refs(vld, indata)
 833
 834         for fgd in get_iterable(indata.get("vnffgd")):
 835             self.validate_fgd_classifiers(fgd)
 836
 837         return indata
 838
 839     @staticmethod
 840     def validate_vld_mgmt_network_with_ip_profile_ref(vld):
 841         if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
 842             raise EngineException("Error at vld[id='{}']:ip-profile-ref"
 843                                   " You cannot set an ip-profile when mgmt-network is True"
 844                                   .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 845
 846     @staticmethod
 847     def validate_vld_connection_point_refs(vld, indata):
 848         for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 849             for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 850                 if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 851                     if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 852                         raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 853                                               "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 854                                               " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 855                                                              constituent_vnfd["member-vnf-index"],
 856                                                              constituent_vnfd["vnfd-id-ref"]),
 857                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 858                     break
 859             else:
 860                 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 861                                       "does not match any constituent-vnfd:member-vnf-index"
 862                                       .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 863                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 864
 865     @staticmethod
 866     def validate_fgd_classifiers(fgd):
 867         for cls in get_iterable(fgd.get("classifier")):
 868             rspref = cls.get("rsp-id-ref")
 869             for rsp in get_iterable(fgd.get("rsp")):
 870                 rspid = rsp.get("id")
 871                 if rspid and rspref and rspid == rspref:
 872                     break
 873             else:
 874                 raise EngineException(
 875                     "Error at vnffgd[id='{}']:classifier[id='{}']:rsp-id-ref '{}' does not match any rsp:id"
 876                     .format(fgd["id"], cls["id"], rspref),
 877                     http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 878
 879     def _validate_input_edit(self, indata, content, force=False):
 880         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 881         """
 882         indata looks as follows:
 883             - In the new case (conformant)
 884                 {'nsdOperationalState': 'DISABLED', 'userDefinedData': {'id': 'string23',
 885                 '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}}
 886             - In the old case (backwards-compatible)
 887                 {'id': 'string23', '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}
 888         """
 889         if "_admin" not in indata:
 890             indata["_admin"] = {}
 891
 892         if "nsdOperationalState" in indata:
 893             if indata["nsdOperationalState"] in ("ENABLED", "DISABLED"):
 894                 indata["_admin"]["operationalState"] = indata.pop("nsdOperationalState")
 895             else:
 896                 raise EngineException("State '{}' is not a valid operational state"
 897                                       .format(indata["nsdOperationalState"]),
 898                                       http_code=HTTPStatus.BAD_REQUEST)
 899
 900         # In the case of user defined data, we need to put the data in the root of the object
 901         # to preserve current expected behaviour
 902         if "userDefinedData" in indata:
 903             data = indata.pop("userDefinedData")
 904             if type(data) == dict:
 905                 indata["_admin"]["userDefinedData"] = data
 906             else:
 907                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 908                                       .format(type(data)),
 909                                       http_code=HTTPStatus.BAD_REQUEST)
 910         if ("operationalState" in indata["_admin"] and
 911                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 912             raise EngineException("nsdOperationalState already {}".format(content["_admin"]["operationalState"]),
 913                                   http_code=HTTPStatus.CONFLICT)
 914         return indata
 915
 916     def _check_descriptor_dependencies(self, session, descriptor):
 917         """
 918         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 919         connection points are ok
 920         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 921         :param descriptor: descriptor to be inserted or edit
 922         :return: None or raises exception
 923         """
 924         if session["force"]:
 925             return
 926         member_vnfd_index = self._get_descriptor_constituent_vnfds_by_member_vnfd_index(session, descriptor)
 927
 928         # Cross references validation in the descriptor and vnfd connection point validation
 929         for vld in get_iterable(descriptor.get("vld")):
 930             self.validate_vld_connection_point_refs_vnfd_connection_points(vld, member_vnfd_index)
 931
 932     def _get_descriptor_constituent_vnfds_by_member_vnfd_index(self, session, descriptor):
 933         member_vnfd_index = {}
 934         if descriptor.get("constituent-vnfd") and not session["force"]:
 935             for vnf in descriptor["constituent-vnfd"]:
 936                 vnfd_id = vnf["vnfd-id-ref"]
 937                 query_filter = self._get_project_filter(session)
 938                 query_filter["id"] = vnfd_id
 939                 vnf_list = self.db.get_list("vnfds", query_filter)
 940                 if not vnf_list:
 941                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 942                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 943
 944                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 945         return member_vnfd_index
 946
 947     @staticmethod
 948     def validate_vld_connection_point_refs_vnfd_connection_points(vld, member_vnfd_index):
 949         for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 950             # look if this vnfd contains this connection point
 951             vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 952             for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 953                 if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 954                     break
 955             else:
 956                 raise EngineException(
 957                     "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 958                     "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 959                     .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 960                             referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 961                     http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 962
 963     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 964         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 965
 966         self._check_descriptor_dependencies(session, final_content)
 967
 968     def check_conflict_on_del(self, session, _id, db_content):
 969         """
 970         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 971         that NSD can be public and be used by other projects.
 972         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 973         :param _id: nsd internal id
 974         :param db_content: The database content of the _id
 975         :return: None or raises EngineException with the conflict
 976         """
 977         if session["force"]:
 978             return
 979         descriptor = db_content
 980         descriptor_id = descriptor.get("id")
 981         if not descriptor_id:  # empty nsd not uploaded
 982             return
 983
 984         # check NSD used by NS
 985         _filter = self._get_project_filter(session)
 986         _filter["nsd-id"] = _id
 987         if self.db.get_list("nsrs", _filter):
 988             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
 989
 990         # check NSD referenced by NST
 991         del _filter["nsd-id"]
 992         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
 993         if self.db.get_list("nsts", _filter):
 994             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
 995                                   http_code=HTTPStatus.CONFLICT)
 996
 997     def sol005_projection(self, data):
 998         data["nsdOnboardingState"] = data["_admin"]["onboardingState"]
 999         data["nsdOperationalState"] = data["_admin"]["operationalState"]
1000         data["nsdUsageState"] = data["_admin"]["usageState"]
1001
1002         links = {}
1003         links["self"] = {"href": "/nsd/v1/ns_descriptors/{}".format(data["_id"])}
1004         links["nsd_content"] = {"href": "/nsd/v1/ns_descriptors/{}/nsd_content".format(data["_id"])}
1005         data["_links"] = links
1006
1007         return super().sol005_projection(data)
1008
1009
1010 class NstTopic(DescriptorTopic):
1011     topic = "nsts"
1012     topic_msg = "nst"
1013     quota_name = "slice_templates"
1014
1015     def __init__(self, db, fs, msg, auth):
1016         DescriptorTopic.__init__(self, db, fs, msg, auth)
1017
1018     @staticmethod
1019     def _remove_envelop(indata=None):
1020         if not indata:
1021             return {}
1022         clean_indata = indata
1023
1024         if clean_indata.get('nst'):
1025             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
1026                 raise EngineException("'nst' must be a list only one element")
1027             clean_indata = clean_indata['nst'][0]
1028         elif clean_indata.get('nst:nst'):
1029             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
1030                 raise EngineException("'nst:nst' must be a list only one element")
1031             clean_indata = clean_indata['nst:nst'][0]
1032         return clean_indata
1033
1034     def _validate_input_new(self, indata, storage_params, force=False):
1035         indata.pop("onboardingState", None)
1036         indata.pop("operationalState", None)
1037         indata.pop("usageState", None)
1038         indata = self.pyangbind_validation("nsts", indata, force)
1039         return indata.copy()
1040
1041     def _check_descriptor_dependencies(self, session, descriptor):
1042         """
1043         Check that the dependent descriptors exist on a new descriptor or edition
1044         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1045         :param descriptor: descriptor to be inserted or edit
1046         :return: None or raises exception
1047         """
1048         if not descriptor.get("netslice-subnet"):
1049             return
1050         for nsd in descriptor["netslice-subnet"]:
1051             nsd_id = nsd["nsd-ref"]
1052             filter_q = self._get_project_filter(session)
1053             filter_q["id"] = nsd_id
1054             if not self.db.get_list("nsds", filter_q):
1055                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
1056                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
1057
1058     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
1059         super().check_conflict_on_edit(session, final_content, edit_content, _id)
1060
1061         self._check_descriptor_dependencies(session, final_content)
1062
1063     def check_conflict_on_del(self, session, _id, db_content):
1064         """
1065         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
1066         that NST can be public and be used by other projects.
1067         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1068         :param _id: nst internal id
1069         :param db_content: The database content of the _id.
1070         :return: None or raises EngineException with the conflict
1071         """
1072         # TODO: Check this method
1073         if session["force"]:
1074             return
1075         # Get Network Slice Template from Database
1076         _filter = self._get_project_filter(session)
1077         _filter["_admin.nst-id"] = _id
1078         if self.db.get_list("nsis", _filter):
1079             raise EngineException("there is at least one Netslice Instance using this descriptor",
1080                                   http_code=HTTPStatus.CONFLICT)
1081
1082     def sol005_projection(self, data):
1083         data["onboardingState"] = data["_admin"]["onboardingState"]
1084         data["operationalState"] = data["_admin"]["operationalState"]
1085         data["usageState"] = data["_admin"]["usageState"]
1086
1087         links = {}
1088         links["self"] = {"href": "/nst/v1/netslice_templates/{}".format(data["_id"])}
1089         links["nst"] = {"href": "/nst/v1/netslice_templates/{}/nst".format(data["_id"])}
1090         data["_links"] = links
1091
1092         return super().sol005_projection(data)
1093
1094
1095 class PduTopic(BaseTopic):
1096     topic = "pdus"
1097     topic_msg = "pdu"
1098     quota_name = "pduds"
1099     schema_new = pdu_new_schema
1100     schema_edit = pdu_edit_schema
1101
1102     def __init__(self, db, fs, msg, auth):
1103         BaseTopic.__init__(self, db, fs, msg, auth)
1104
1105     @staticmethod
1106     def format_on_new(content, project_id=None, make_public=False):
1107         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
1108         content["_admin"]["onboardingState"] = "CREATED"
1109         content["_admin"]["operationalState"] = "ENABLED"
1110         content["_admin"]["usageState"] = "NOT_IN_USE"
1111
1112     def check_conflict_on_del(self, session, _id, db_content):
1113         """
1114         Check that there is not any vnfr that uses this PDU
1115         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1116         :param _id: pdu internal id
1117         :param db_content: The database content of the _id.
1118         :return: None or raises EngineException with the conflict
1119         """
1120         if session["force"]:
1121             return
1122
1123         _filter = self._get_project_filter(session)
1124         _filter["vdur.pdu-id"] = _id
1125         if self.db.get_list("vnfrs", _filter):
1126             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)
1127
1128
1129 class VnfPkgOpTopic(BaseTopic):
1130     topic = "vnfpkgops"
1131     topic_msg = "vnfd"
1132     schema_new = vnfpkgop_new_schema
1133     schema_edit = None
1134
1135     def __init__(self, db, fs, msg, auth):
1136         BaseTopic.__init__(self, db, fs, msg, auth)
1137
1138     def edit(self, session, _id, indata=None, kwargs=None, content=None):
1139         raise EngineException("Method 'edit' not allowed for topic '{}'".format(self.topic),
1140                               HTTPStatus.METHOD_NOT_ALLOWED)
1141
1142     def delete(self, session, _id, dry_run=False):
1143         raise EngineException("Method 'delete' not allowed for topic '{}'".format(self.topic),
1144                               HTTPStatus.METHOD_NOT_ALLOWED)
1145
1146     def delete_list(self, session, filter_q=None):
1147         raise EngineException("Method 'delete_list' not allowed for topic '{}'".format(self.topic),
1148                               HTTPStatus.METHOD_NOT_ALLOWED)
1149
1150     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
1151         """
1152         Creates a new entry into database.
1153         :param rollback: list to append created items at database in case a rollback may to be done
1154         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1155         :param indata: data to be inserted
1156         :param kwargs: used to override the indata descriptor
1157         :param headers: http request headers
1158         :return: _id, op_id:
1159             _id: identity of the inserted data.
1160              op_id: None
1161         """
1162         self._update_input_with_kwargs(indata, kwargs)
1163         validate_input(indata, self.schema_new)
1164         vnfpkg_id = indata["vnfPkgId"]
1165         filter_q = BaseTopic._get_project_filter(session)
1166         filter_q["_id"] = vnfpkg_id
1167         vnfd = self.db.get_one("vnfds", filter_q)
1168         operation = indata["lcmOperationType"]
1169         kdu_name = indata["kdu_name"]
1170         for kdu in vnfd.get("kdu", []):
1171             if kdu["name"] == kdu_name:
1172                 helm_chart = kdu.get("helm-chart")
1173                 juju_bundle = kdu.get("juju-bundle")
1174                 break
1175         else:
1176             raise EngineException("Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id, kdu_name))
1177         if helm_chart:
1178             indata["helm-chart"] = helm_chart
1179             match = fullmatch(r"([^/]*)/([^/]*)", helm_chart)
1180             repo_name = match.group(1) if match else None
1181         elif juju_bundle:
1182             indata["juju-bundle"] = juju_bundle
1183             match = fullmatch(r"([^/]*)/([^/]*)", juju_bundle)
1184             repo_name = match.group(1) if match else None
1185         else:
1186             raise EngineException("Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']"
1187                                   .format(vnfpkg_id, kdu_name))
1188         if repo_name:
1189             del filter_q["_id"]
1190             filter_q["name"] = repo_name
1191             repo = self.db.get_one("k8srepos", filter_q)
1192             k8srepo_id = repo.get("_id")
1193             k8srepo_url = repo.get("url")
1194         else:
1195             k8srepo_id = None
1196             k8srepo_url = None
1197         indata["k8srepoId"] = k8srepo_id
1198         indata["k8srepo_url"] = k8srepo_url
1199         vnfpkgop_id = str(uuid4())
1200         vnfpkgop_desc = {
1201             "_id": vnfpkgop_id,
1202             "operationState": "PROCESSING",
1203             "vnfPkgId": vnfpkg_id,
1204             "lcmOperationType": operation,
1205             "isAutomaticInvocation": False,
1206             "isCancelPending": False,
1207             "operationParams": indata,
1208             "links": {
1209                 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id,
1210                 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id,
1211             }
1212         }
1213         self.format_on_new(vnfpkgop_desc, session["project_id"], make_public=session["public"])
1214         ctime = vnfpkgop_desc["_admin"]["created"]
1215         vnfpkgop_desc["statusEnteredTime"] = ctime
1216         vnfpkgop_desc["startTime"] = ctime
1217         self.db.create(self.topic, vnfpkgop_desc)
1218         rollback.append({"topic": self.topic, "_id": vnfpkgop_id})
1219         self.msg.write(self.topic_msg, operation, vnfpkgop_desc)
1220         return vnfpkgop_id, None