osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 import importlib
  20 # import logging
  21 from hashlib import md5
  22 from osm_common.dbbase import DbException, deep_update_rfc7396
  23 from http import HTTPStatus
  24 from time import time
  25 from uuid import uuid4
  26 from re import fullmatch
  27 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema, \
  28     validate_input, vnfpkgop_new_schema
  29 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  30 etsi_nfv_vnfd = importlib.import_module("osm_im.etsi-nfv-vnfd")
  31 etsi_nfv_nsd = importlib.import_module("osm_im.etsi-nfv-nsd")
  32 from osm_im.nst import nst as nst_im
  33 from pyangbind.lib.serialise import pybindJSONDecoder
  34 import pyangbind.lib.pybindJSON as pybindJSON
  35
  36 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  37
  38
  39 class DescriptorTopic(BaseTopic):
  40
  41     def __init__(self, db, fs, msg, auth):
  42         BaseTopic.__init__(self, db, fs, msg, auth)
  43
  44     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  45         super().check_conflict_on_edit(session, final_content, edit_content, _id)
  46
  47         def _check_unique_id_name(descriptor, position=""):
  48             for desc_key, desc_item in descriptor.items():
  49                 if isinstance(desc_item, list) and desc_item:
  50                     used_ids = []
  51                     desc_item_id = None
  52                     for index, list_item in enumerate(desc_item):
  53                         if isinstance(list_item, dict):
  54                             _check_unique_id_name(list_item, "{}.{}[{}]"
  55                                                   .format(position, desc_key, index))
  56                             # Base case
  57                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  58                                 desc_item_id = "id" if list_item.get("id") else "name"
  59                             if desc_item_id and list_item.get(desc_item_id):
  60                                 if list_item[desc_item_id] in used_ids:
  61                                     position = "{}.{}[{}]".format(position, desc_key, index)
  62                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  63                                                           .format(desc_item_id, list_item[desc_item_id],
  64                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  65                                 used_ids.append(list_item[desc_item_id])
  66
  67         _check_unique_id_name(final_content)
  68         # 1. validate again with pyangbind
  69         # 1.1. remove internal keys
  70         internal_keys = {}
  71         for k in ("_id", "_admin"):
  72             if k in final_content:
  73                 internal_keys[k] = final_content.pop(k)
  74         storage_params = internal_keys["_admin"].get("storage")
  75         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  76         # 1.2. modify final_content with a serialized version
  77         final_content.clear()
  78         final_content.update(serialized)
  79         # 1.3. restore internal keys
  80         for k, v in internal_keys.items():
  81             final_content[k] = v
  82         if session["force"]:
  83             return
  84         # 2. check that this id is not present
  85         if "id" in edit_content:
  86             _filter = self._get_project_filter(session)
  87             _filter["id"] = final_content["id"]
  88             _filter["_id.neq"] = _id
  89             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  90                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  91                                                                                                final_content["id"]),
  92                                       HTTPStatus.CONFLICT)
  93
  94     @staticmethod
  95     def format_on_new(content, project_id=None, make_public=False):
  96         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  97         content["_admin"]["onboardingState"] = "CREATED"
  98         content["_admin"]["operationalState"] = "DISABLED"
  99         content["_admin"]["usageState"] = "NOT_IN_USE"
 100
 101     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 102         """
 103         Deletes file system storage associated with the descriptor
 104         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 105         :param _id: server internal id
 106         :param db_content: The database content of the descriptor
 107         :param not_send_msg: To not send message (False) or store content (list) instead
 108         :return: None if ok or raises EngineException with the problem
 109         """
 110         self.fs.file_delete(_id, ignore_non_exist=True)
 111         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 112
 113     @staticmethod
 114     def get_one_by_id(db, session, topic, id):
 115         # find owned by this project
 116         _filter = BaseTopic._get_project_filter(session)
 117         _filter["id"] = id
 118         desc_list = db.get_list(topic, _filter)
 119         if len(desc_list) == 1:
 120             return desc_list[0]
 121         elif len(desc_list) > 1:
 122             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 123                               HTTPStatus.CONFLICT)
 124
 125         # not found any: try to find public
 126         _filter = BaseTopic._get_project_filter(session)
 127         _filter["id"] = id
 128         desc_list = db.get_list(topic, _filter)
 129         if not desc_list:
 130             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 131         elif len(desc_list) == 1:
 132             return desc_list[0]
 133         else:
 134             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 135                 topic[:-1], id), HTTPStatus.CONFLICT)
 136
 137     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 138         """
 139         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 140         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 141         (self.upload_content)
 142         :param rollback: list to append created items at database in case a rollback may to be done
 143         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 144         :param indata: data to be inserted
 145         :param kwargs: used to override the indata descriptor
 146         :param headers: http request headers
 147         :return: _id, None: identity of the inserted data; and None as there is not any operation
 148         """
 149
 150         # No needed to capture exceptions
 151         # Check Quota
 152         self.check_quota(session)
 153
 154         # _remove_envelop
 155         if indata:
 156             if "userDefinedData" in indata:
 157                 indata = indata['userDefinedData']
 158
 159         # Override descriptor with query string kwargs
 160         self._update_input_with_kwargs(indata, kwargs)
 161         # uncomment when this method is implemented.
 162         # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 163         # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 164
 165         content = {"_admin": {"userDefinedData": indata}}
 166         self.format_on_new(content, session["project_id"], make_public=session["public"])
 167         _id = self.db.create(self.topic, content)
 168         rollback.append({"topic": self.topic, "_id": _id})
 169         self._send_msg("created", {"_id": _id})
 170         return _id, None
 171
 172     def upload_content(self, session, _id, indata, kwargs, headers):
 173         """
 174         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 175         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 176         :param _id : the nsd,vnfd is already created, this is the id
 177         :param indata: http body request
 178         :param kwargs: user query string to override parameters. NOT USED
 179         :param headers:  http request headers
 180         :return: True if package is completely uploaded or False if partial content has been uploded
 181             Raise exception on error
 182         """
 183         # Check that _id exists and it is valid
 184         current_desc = self.show(session, _id)
 185
 186         content_range_text = headers.get("Content-Range")
 187         expected_md5 = headers.get("Content-File-MD5")
 188         compressed = None
 189         content_type = headers.get("Content-Type")
 190         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 191                 "application/zip" in content_type:
 192             compressed = "gzip"
 193         filename = headers.get("Content-Filename")
 194         if not filename:
 195             filename = "package.tar.gz" if compressed else "package"
 196         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 197         file_pkg = None
 198         error_text = ""
 199         try:
 200             if content_range_text:
 201                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 202                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 203                     raise IndexError()
 204                 start = int(content_range[1])
 205                 end = int(content_range[2]) + 1
 206                 total = int(content_range[3])
 207             else:
 208                 start = 0
 209             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 210
 211             if start:
 212                 if not self.fs.file_exists(temp_folder, 'dir'):
 213                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 214             else:
 215                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 216                 self.fs.mkdir(temp_folder)
 217
 218             storage = self.fs.get_params()
 219             storage["folder"] = _id
 220
 221             file_path = (temp_folder, filename)
 222             if self.fs.file_exists(file_path, 'file'):
 223                 file_size = self.fs.file_size(file_path)
 224             else:
 225                 file_size = 0
 226             if file_size != start:
 227                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 228                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 229             file_pkg = self.fs.file_open(file_path, 'a+b')
 230             if isinstance(indata, dict):
 231                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 232                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 233             else:
 234                 indata_len = 0
 235                 while True:
 236                     indata_text = indata.read(4096)
 237                     indata_len += len(indata_text)
 238                     if not indata_text:
 239                         break
 240                     file_pkg.write(indata_text)
 241             if content_range_text:
 242                 if indata_len != end - start:
 243                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 244                         start, end - 1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 245                 if end != total:
 246                     # TODO update to UPLOADING
 247                     return False
 248
 249             # PACKAGE UPLOADED
 250             if expected_md5:
 251                 file_pkg.seek(0, 0)
 252                 file_md5 = md5()
 253                 chunk_data = file_pkg.read(1024)
 254                 while chunk_data:
 255                     file_md5.update(chunk_data)
 256                     chunk_data = file_pkg.read(1024)
 257                 if expected_md5 != file_md5.hexdigest():
 258                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 259             file_pkg.seek(0, 0)
 260             if compressed == "gzip":
 261                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 262                 descriptor_file_name = None
 263                 for tarinfo in tar:
 264                     tarname = tarinfo.name
 265                     tarname_path = tarname.split("/")
 266                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 267                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 268                     if len(tarname_path) == 1 and not tarinfo.isdir():
 269                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 270                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 271                         storage["pkg-dir"] = tarname_path[0]
 272                         if len(tarname_path) == 2:
 273                             if descriptor_file_name:
 274                                 raise EngineException(
 275                                     "Found more than one descriptor file at package descriptor tar.gz")
 276                             descriptor_file_name = tarname
 277                 if not descriptor_file_name:
 278                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 279                 storage["descriptor"] = descriptor_file_name
 280                 storage["zipfile"] = filename
 281                 self.fs.file_extract(tar, temp_folder)
 282                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 283                     content = descriptor_file.read()
 284             else:
 285                 content = file_pkg.read()
 286                 storage["descriptor"] = descriptor_file_name = filename
 287
 288             if descriptor_file_name.endswith(".json"):
 289                 error_text = "Invalid json format "
 290                 indata = json.load(content)
 291             else:
 292                 error_text = "Invalid yaml format "
 293                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 294
 295             current_desc["_admin"]["storage"] = storage
 296             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 297             current_desc["_admin"]["operationalState"] = "ENABLED"
 298
 299             indata = self._remove_envelop(indata)
 300
 301             # Override descriptor with query string kwargs
 302             if kwargs:
 303                 self._update_input_with_kwargs(indata, kwargs)
 304
 305             deep_update_rfc7396(current_desc, indata)
 306             self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 307             current_desc["_admin"]["modified"] = time()
 308             self.db.replace(self.topic, _id, current_desc)
 309             self.fs.dir_rename(temp_folder, _id)
 310
 311             indata["_id"] = _id
 312             self._send_msg("edited", indata)
 313
 314             # TODO if descriptor has changed because kwargs update content and remove cached zip
 315             # TODO if zip is not present creates one
 316             return True
 317
 318         except EngineException:
 319             raise
 320         except IndexError:
 321             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 322                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 323         except IOError as e:
 324             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 325         except tarfile.ReadError as e:
 326             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 327         except (ValueError, yaml.YAMLError) as e:
 328             raise EngineException(error_text + str(e))
 329         except ValidationError as e:
 330             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 331         finally:
 332             if file_pkg:
 333                 file_pkg.close()
 334
 335     def get_file(self, session, _id, path=None, accept_header=None):
 336         """
 337         Return the file content of a vnfd or nsd
 338         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 339         :param _id: Identity of the vnfd, nsd
 340         :param path: artifact path or "$DESCRIPTOR" or None
 341         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 342         :return: opened file plus Accept format or raises an exception
 343         """
 344         accept_text = accept_zip = False
 345         if accept_header:
 346             if 'text/plain' in accept_header or '*/*' in accept_header:
 347                 accept_text = True
 348             if 'application/zip' in accept_header or '*/*' in accept_header:
 349                 accept_zip = 'application/zip'
 350             elif 'application/gzip' in accept_header:
 351                 accept_zip = 'application/gzip'
 352
 353         if not accept_text and not accept_zip:
 354             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 355                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 356
 357         content = self.show(session, _id)
 358         if content["_admin"]["onboardingState"] != "ONBOARDED":
 359             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 360                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 361                                   http_code=HTTPStatus.CONFLICT)
 362         storage = content["_admin"]["storage"]
 363         if path is not None and path != "$DESCRIPTOR":  # artifacts
 364             if not storage.get('pkg-dir'):
 365                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 366             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 367                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 368                 return folder_content, "text/plain"
 369                 # TODO manage folders in http
 370             else:
 371                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \
 372                     "application/octet-stream"
 373
 374         # pkgtype   accept  ZIP  TEXT    -> result
 375         # manyfiles         yes  X       -> zip
 376         #                   no   yes     -> error
 377         # onefile           yes  no      -> zip
 378         #                   X    yes     -> text
 379         contain_many_files = False
 380         if storage.get('pkg-dir'):
 381             # check if there are more than one file in the package, ignoring checksums.txt.
 382             pkg_files = self.fs.dir_ls((storage['folder'], storage['pkg-dir']))
 383             if len(pkg_files) >= 3 or (len(pkg_files) == 2 and 'checksums.txt' not in pkg_files):
 384                 contain_many_files = True
 385         if accept_text and (not contain_many_files or path == "$DESCRIPTOR"):
 386             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 387         elif contain_many_files and not accept_zip:
 388             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 389                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 390         else:
 391             if not storage.get('zipfile'):
 392                 # TODO generate zipfile if not present
 393                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 394                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 395             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 396
 397     def _remove_yang_prefixes_from_descriptor(self, descriptor):
 398         new_descriptor = {}
 399         for k, v in descriptor.items():
 400             new_v = v
 401             if isinstance(v, dict):
 402                 new_v = self._remove_yang_prefixes_from_descriptor(v)
 403             elif isinstance(v, list):
 404                 new_v = list()
 405                 for x in v:
 406                     if isinstance(x, dict):
 407                         new_v.append(self._remove_yang_prefixes_from_descriptor(x))
 408                     else:
 409                         new_v.append(x)
 410             new_descriptor[k.split(':')[-1]] = new_v
 411         return new_descriptor
 412
 413     def pyangbind_validation(self, item, data, force=False):
 414         raise EngineException("Not possible to validate '{}' item".format(item),
 415                               http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 416
 417     def _validate_input_edit(self, indata, content, force=False):
 418         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 419         if "_id" in indata:
 420             indata.pop("_id")
 421         if "_admin" not in indata:
 422             indata["_admin"] = {}
 423
 424         if "operationalState" in indata:
 425             if indata["operationalState"] in ("ENABLED", "DISABLED"):
 426                 indata["_admin"]["operationalState"] = indata.pop("operationalState")
 427             else:
 428                 raise EngineException("State '{}' is not a valid operational state"
 429                                       .format(indata["operationalState"]),
 430                                       http_code=HTTPStatus.BAD_REQUEST)
 431
 432         # In the case of user defined data, we need to put the data in the root of the object
 433         # to preserve current expected behaviour
 434         if "userDefinedData" in indata:
 435             data = indata.pop("userDefinedData")
 436             if type(data) == dict:
 437                 indata["_admin"]["userDefinedData"] = data
 438             else:
 439                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 440                                       .format(type(data)),
 441                                       http_code=HTTPStatus.BAD_REQUEST)
 442
 443         if ("operationalState" in indata["_admin"] and
 444                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 445             raise EngineException("operationalState already {}".format(content["_admin"]["operationalState"]),
 446                                   http_code=HTTPStatus.CONFLICT)
 447
 448         return indata
 449
 450
 451 class VnfdTopic(DescriptorTopic):
 452     topic = "vnfds"
 453     topic_msg = "vnfd"
 454
 455     def __init__(self, db, fs, msg, auth):
 456         DescriptorTopic.__init__(self, db, fs, msg, auth)
 457
 458     def pyangbind_validation(self, item, data, force=False):
 459         if self._descriptor_data_is_in_old_format(data):
 460             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 461                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 462         try:
 463             virtual_compute_descriptors = data.get('virtual-compute-desc')
 464             virtual_storage_descriptors = data.get('virtual-storage-desc')
 465             myvnfd = etsi_nfv_vnfd.etsi_nfv_vnfd()
 466             pybindJSONDecoder.load_ietf_json({'etsi-nfv-vnfd:vnfd': data}, None, None, obj=myvnfd,
 467                                              path_helper=True, skip_unknown=force)
 468             out = pybindJSON.dumps(myvnfd, mode="ietf")
 469             desc_out = self._remove_envelop(yaml.safe_load(out))
 470             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 471             if virtual_compute_descriptors:
 472                 desc_out['virtual-compute-desc'] = virtual_compute_descriptors
 473             if virtual_storage_descriptors:
 474                 desc_out['virtual-storage-desc'] = virtual_storage_descriptors
 475             return desc_out
 476         except Exception as e:
 477             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 478                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 479
 480     @staticmethod
 481     def _descriptor_data_is_in_old_format(data):
 482         return ('vnfd-catalog' in data) or ('vnfd:vnfd-catalog' in data)
 483
 484     @staticmethod
 485     def _remove_envelop(indata=None):
 486         if not indata:
 487             return {}
 488         clean_indata = indata
 489
 490         if clean_indata.get('etsi-nfv-vnfd:vnfd'):
 491             if not isinstance(clean_indata['etsi-nfv-vnfd:vnfd'], dict):
 492                 raise EngineException("'etsi-nfv-vnfd:vnfd' must be a dict")
 493             clean_indata = clean_indata['etsi-nfv-vnfd:vnfd']
 494         elif clean_indata.get('vnfd'):
 495             if not isinstance(clean_indata['vnfd'], dict):
 496                 raise EngineException("'vnfd' must be dict")
 497             clean_indata = clean_indata['vnfd']
 498
 499         return clean_indata
 500
 501     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 502         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 503
 504         # set type of vnfd
 505         contains_pdu = False
 506         contains_vdu = False
 507         for vdu in get_iterable(final_content.get("vdu")):
 508             if vdu.get("pdu-type"):
 509                 contains_pdu = True
 510             else:
 511                 contains_vdu = True
 512         if contains_pdu:
 513             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 514         elif contains_vdu:
 515             final_content["_admin"]["type"] = "vnfd"
 516         # if neither vud nor pdu do not fill type
 517
 518     def check_conflict_on_del(self, session, _id, db_content):
 519         """
 520         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 521         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 522         that uses this vnfd
 523         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 524         :param _id: vnfd internal id
 525         :param db_content: The database content of the _id.
 526         :return: None or raises EngineException with the conflict
 527         """
 528         if session["force"]:
 529             return
 530         descriptor = db_content
 531         descriptor_id = descriptor.get("id")
 532         if not descriptor_id:  # empty vnfd not uploaded
 533             return
 534
 535         _filter = self._get_project_filter(session)
 536
 537         # check vnfrs using this vnfd
 538         _filter["vnfd-id"] = _id
 539         if self.db.get_list("vnfrs", _filter):
 540             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 541
 542         # check NSD referencing this VNFD
 543         del _filter["vnfd-id"]
 544         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 545         if self.db.get_list("nsds", _filter):
 546             raise EngineException("There is at least one NSD referencing this descriptor",
 547                                   http_code=HTTPStatus.CONFLICT)
 548
 549     def _validate_input_new(self, indata, storage_params, force=False):
 550         indata.pop("onboardingState", None)
 551         indata.pop("operationalState", None)
 552         indata.pop("usageState", None)
 553         indata.pop("links", None)
 554
 555         indata = self.pyangbind_validation("vnfds", indata, force)
 556         # Cross references validation in the descriptor
 557
 558         self.validate_mgmt_interface_connection_point(indata)
 559
 560         for vdu in get_iterable(indata.get("vdu")):
 561             self.validate_vdu_internal_connection_points(vdu)
 562             self._validate_vdu_charms_in_package(storage_params, vdu, indata)
 563             self._validate_vdu_cloud_init_in_package(storage_params, vdu, indata)
 564
 565         self._validate_vnf_charms_in_package(storage_params, indata)
 566
 567         self.validate_external_connection_points(indata)
 568         self.validate_internal_virtual_links(indata)
 569         self.validate_monitoring_params(indata)
 570         self.validate_scaling_group_descriptor(indata)
 571
 572         return indata
 573
 574     @staticmethod
 575     def validate_mgmt_interface_connection_point(indata):
 576         if not indata.get("vdu"):
 577             return
 578         if not indata.get("mgmt-cp"):
 579             raise EngineException("'mgmt-cp' is a mandatory field and it is not defined",
 580                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 581
 582         for cp in get_iterable(indata.get("ext-cpd")):
 583             if cp["id"] == indata["mgmt-cp"]:
 584                 break
 585         else:
 586             raise EngineException("mgmt-cp='{}' must match an existing ext-cpd".format(indata["mgmt-cp"]),
 587                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 588
 589     @staticmethod
 590     def validate_vdu_internal_connection_points(vdu):
 591         int_cpds = set()
 592         for cpd in get_iterable(vdu.get("int-cpd")):
 593             cpd_id = cpd.get("id")
 594             if cpd_id and cpd_id in int_cpds:
 595                 raise EngineException("vdu[id='{}']:int-cpd[id='{}'] is already used by other int-cpd"
 596                                       .format(vdu["id"], cpd_id),
 597                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 598             int_cpds.add(cpd_id)
 599
 600     @staticmethod
 601     def validate_external_connection_points(indata):
 602         all_vdus_int_cpds = set()
 603         for vdu in get_iterable(indata.get("vdu")):
 604             for int_cpd in get_iterable(vdu.get("int-cpd")):
 605                 all_vdus_int_cpds.add((vdu.get("id"), int_cpd.get("id")))
 606
 607         ext_cpds = set()
 608         for cpd in get_iterable(indata.get("ext-cpd")):
 609             cpd_id = cpd.get("id")
 610             if cpd_id and cpd_id in ext_cpds:
 611                 raise EngineException("ext-cpd[id='{}'] is already used by other ext-cpd".format(cpd_id),
 612                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 613             ext_cpds.add(cpd_id)
 614
 615             int_cpd = cpd.get("int-cpd")
 616             if int_cpd:
 617                 if (int_cpd.get("vdu-id"), int_cpd.get("cpd")) not in all_vdus_int_cpds:
 618                     raise EngineException("ext-cpd[id='{}']:int-cpd must match an existing vdu int-cpd".format(cpd_id),
 619                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 620             # TODO: Validate k8s-cluster-net points to a valid k8s-cluster:nets ?
 621
 622     def _validate_vdu_charms_in_package(self, storage_params, vdu, indata):
 623         if not vdu.get("vdu-configuration"):
 624             return
 625         for vdu_configuration in get_iterable(indata.get("vdu-configuration")):
 626             if vdu_configuration.get("juju"):
 627                 if not self._validate_package_folders(storage_params, 'charms'):
 628                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 629                                           "package".format(indata["id"]))
 630
 631     def _validate_vdu_cloud_init_in_package(self, storage_params, vdu, indata):
 632         if not vdu.get("cloud-init-file"):
 633             return
 634         if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 635             raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 636                                   "package".format(indata["id"], vdu["id"]))
 637
 638     def _validate_vnf_charms_in_package(self, storage_params, indata):
 639         if not indata.get("vnf-configuration"):
 640             return
 641         for vnf_configuration in get_iterable(indata.get("vnf-configuration")):
 642             if vnf_configuration.get("juju"):
 643                 if not self._validate_package_folders(storage_params, 'charms'):
 644                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 645                                           "package".format(indata["id"]))
 646
 647     def _validate_package_folders(self, storage_params, folder, file=None):
 648         if not storage_params or not storage_params.get("pkg-dir"):
 649             return False
 650         else:
 651             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 652                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 653             else:
 654                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 655             if file:
 656                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 657             else:
 658                 if self.fs.file_exists(f, 'dir'):
 659                     if self.fs.dir_ls(f):
 660                         return True
 661             return False
 662
 663     @staticmethod
 664     def validate_internal_virtual_links(indata):
 665         all_ivld_ids = set()
 666         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 667             ivld_id = ivld.get("id")
 668             if ivld_id and ivld_id in all_ivld_ids:
 669                 raise EngineException("Duplicated VLD id in int-virtual-link-desc[id={}]".format(ivld_id),
 670                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 671             else:
 672                 all_ivld_ids.add(ivld_id)
 673
 674         for vdu in get_iterable(indata.get("vdu")):
 675             for int_cpd in get_iterable(vdu.get("int-cpd")):
 676                 int_cpd_ivld_id = int_cpd.get("int-virtual-link-desc")
 677                 if int_cpd_ivld_id and int_cpd_ivld_id not in all_ivld_ids:
 678                     raise EngineException(
 679                         "vdu[id='{}']:int-cpd[id='{}']:int-virtual-link-desc='{}' must match an existing "
 680                         "int-virtual-link-desc".format(vdu["id"], int_cpd["id"], int_cpd_ivld_id),
 681                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 682
 683         for df in get_iterable(indata.get("df")):
 684             for vlp in get_iterable(df.get("virtual-link-profile")):
 685                 vlp_ivld_id = vlp.get("id")
 686                 if vlp_ivld_id and vlp_ivld_id not in all_ivld_ids:
 687                     raise EngineException("df[id='{}']:virtual-link-profile='{}' must match an existing "
 688                                           "int-virtual-link-desc".format(df["id"], vlp_ivld_id),
 689                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 690
 691     @staticmethod
 692     def validate_monitoring_params(indata):
 693         all_monitoring_params = set()
 694         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 695             for mp in get_iterable(ivld.get("monitoring-parameters")):
 696                 mp_id = mp.get("id")
 697                 if mp_id and mp_id in all_monitoring_params:
 698                     raise EngineException("Duplicated monitoring-parameter id in "
 699                                           "int-virtual-link-desc[id='{}']:monitoring-parameters[id='{}']"
 700                                           .format(ivld["id"], mp_id),
 701                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 702                 else:
 703                     all_monitoring_params.add(mp_id)
 704
 705         for vdu in get_iterable(indata.get("vdu")):
 706             for mp in get_iterable(vdu.get("monitoring-parameter")):
 707                 mp_id = mp.get("id")
 708                 if mp_id and mp_id in all_monitoring_params:
 709                     raise EngineException("Duplicated monitoring-parameter id in "
 710                                           "vdu[id='{}']:monitoring-parameter[id='{}']"
 711                                           .format(vdu["id"], mp_id),
 712                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 713                 else:
 714                     all_monitoring_params.add(mp_id)
 715
 716         for df in get_iterable(indata.get("df")):
 717             for mp in get_iterable(df.get("monitoring-parameter")):
 718                 mp_id = mp.get("id")
 719                 if mp_id and mp_id in all_monitoring_params:
 720                     raise EngineException("Duplicated monitoring-parameter id in "
 721                                           "df[id='{}']:monitoring-parameter[id='{}']"
 722                                           .format(df["id"], mp_id),
 723                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 724                 else:
 725                     all_monitoring_params.add(mp_id)
 726
 727     @staticmethod
 728     def validate_scaling_group_descriptor(indata):
 729         all_monitoring_params = set()
 730         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 731             for mp in get_iterable(ivld.get("monitoring-parameters")):
 732                 all_monitoring_params.add(mp.get("id"))
 733
 734         for vdu in get_iterable(indata.get("vdu")):
 735             for mp in get_iterable(vdu.get("monitoring-parameter")):
 736                 all_monitoring_params.add(mp.get("id"))
 737
 738         for df in get_iterable(indata.get("df")):
 739             for mp in get_iterable(df.get("monitoring-parameter")):
 740                 all_monitoring_params.add(mp.get("id"))
 741
 742         for df in get_iterable(indata.get("df")):
 743             for sa in get_iterable(df.get("scaling-aspect")):
 744                 for sp in get_iterable(sa.get("scaling-policy")):
 745                     for sc in get_iterable(sp.get("scaling-criteria")):
 746                         sc_monitoring_param = sc.get("vnf-monitoring-param-ref")
 747                         if sc_monitoring_param and sc_monitoring_param not in all_monitoring_params:
 748                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-policy"
 749                                                   "[name='{}']:scaling-criteria[name='{}']: "
 750                                                   "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 751                                                   .format(df["id"], sa["id"], sp["name"], sc["name"],
 752                                                           sc_monitoring_param),
 753                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 754
 755                 for sca in get_iterable(sa.get("scaling-config-action")):
 756                     if not indata.get("vnf-configuration"):
 757                         raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced "
 758                                               "by df[id='{}']:scaling-aspect[id='{}']:scaling-config-action"
 759                                               .format(df["id"], sa["id"]),
 760                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 761                     for configuration in get_iterable(indata["vnf-configuration"]):
 762                         for primitive in get_iterable(configuration.get("config-primitive")):
 763                             if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 764                                 break
 765                         else:
 766                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-config-action:vnf-"
 767                                                   "config-primitive-name-ref='{}' does not match any "
 768                                                   "vnf-configuration:config-primitive:name"
 769                                                   .format(df["id"], sa["id"], sca["vnf-config-primitive-name-ref"]),
 770                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 771
 772     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 773         """
 774         Deletes associate file system storage (via super)
 775         Deletes associated vnfpkgops from database.
 776         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 777         :param _id: server internal id
 778         :param db_content: The database content of the descriptor
 779         :return: None
 780         :raises: FsException in case of error while deleting associated storage
 781         """
 782         super().delete_extra(session, _id, db_content, not_send_msg)
 783         self.db.del_list("vnfpkgops", {"vnfPkgId": _id})
 784
 785     def sol005_projection(self, data):
 786         data["onboardingState"] = data["_admin"]["onboardingState"]
 787         data["operationalState"] = data["_admin"]["operationalState"]
 788         data["usageState"] = data["_admin"]["usageState"]
 789
 790         links = {}
 791         links["self"] = {"href": "/vnfpkgm/v1/vnf_packages/{}".format(data["_id"])}
 792         links["vnfd"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(data["_id"])}
 793         links["packageContent"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/package_content".format(data["_id"])}
 794         data["_links"] = links
 795
 796         return super().sol005_projection(data)
 797
 798
 799 class NsdTopic(DescriptorTopic):
 800     topic = "nsds"
 801     topic_msg = "nsd"
 802
 803     def __init__(self, db, fs, msg, auth):
 804         DescriptorTopic.__init__(self, db, fs, msg, auth)
 805
 806     def pyangbind_validation(self, item, data, force=False):
 807         if self._descriptor_data_is_in_old_format(data):
 808             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 809                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 810         try:
 811             mynsd = etsi_nfv_nsd.etsi_nfv_nsd()
 812             pybindJSONDecoder.load_ietf_json({'nsd': {'nsd': [data]}}, None, None, obj=mynsd,
 813                                              path_helper=True, skip_unknown=force)
 814             out = pybindJSON.dumps(mynsd, mode="ietf")
 815             desc_out = self._remove_envelop(yaml.safe_load(out))
 816             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 817             return desc_out
 818         except Exception as e:
 819             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 820                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 821
 822     @staticmethod
 823     def _descriptor_data_is_in_old_format(data):
 824         return ('nsd-catalog' in data) or ('nsd:nsd-catalog' in data)
 825
 826     @staticmethod
 827     def _remove_envelop(indata=None):
 828         if not indata:
 829             return {}
 830         clean_indata = indata
 831
 832         if clean_indata.get('nsd'):
 833             clean_indata = clean_indata['nsd']
 834         elif clean_indata.get('etsi-nfv-nsd:nsd'):
 835             clean_indata = clean_indata['etsi-nfv-nsd:nsd']
 836         if clean_indata.get('nsd'):
 837             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 838                 raise EngineException("'nsd' must be a list of only one element")
 839             clean_indata = clean_indata['nsd'][0]
 840         return clean_indata
 841
 842     def _validate_input_new(self, indata, storage_params, force=False):
 843         indata.pop("nsdOnboardingState", None)
 844         indata.pop("nsdOperationalState", None)
 845         indata.pop("nsdUsageState", None)
 846
 847         indata.pop("links", None)
 848
 849         indata = self.pyangbind_validation("nsds", indata, force)
 850         # Cross references validation in the descriptor
 851         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 852         for vld in get_iterable(indata.get("virtual-link-desc")):
 853             self.validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata)
 854
 855         self.validate_vnf_profiles_vnfd_id(indata)
 856
 857         return indata
 858
 859     @staticmethod
 860     def validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata):
 861         if not vld.get("mgmt-network"):
 862             return
 863         vld_id = vld.get("id")
 864         for df in get_iterable(indata.get("df")):
 865             for vlp in get_iterable(df.get("virtual-link-profile")):
 866                 if vld_id and vld_id == vlp.get("virtual-link-desc-id"):
 867                     if vlp.get("virtual-link-protocol-data"):
 868                         raise EngineException("Error at df[id='{}']:virtual-link-profile[id='{}']:virtual-link-"
 869                                               "protocol-data You cannot set a virtual-link-protocol-data "
 870                                               "when mgmt-network is True"
 871                                               .format(df["id"], vlp["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 872
 873     @staticmethod
 874     def validate_vnf_profiles_vnfd_id(indata):
 875         all_vnfd_ids = set(get_iterable(indata.get("vnfd-id")))
 876         for df in get_iterable(indata.get("df")):
 877             for vnf_profile in get_iterable(df.get("vnf-profile")):
 878                 vnfd_id = vnf_profile.get("vnfd-id")
 879                 if vnfd_id and vnfd_id not in all_vnfd_ids:
 880                     raise EngineException("Error at df[id='{}']:vnf_profile[id='{}']:vnfd-id='{}' "
 881                                           "does not match any vnfd-id".format(df["id"], vnf_profile["id"], vnfd_id),
 882                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 883
 884     def _validate_input_edit(self, indata, content, force=False):
 885         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 886         """
 887         indata looks as follows:
 888             - In the new case (conformant)
 889                 {'nsdOperationalState': 'DISABLED', 'userDefinedData': {'id': 'string23',
 890                 '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}}
 891             - In the old case (backwards-compatible)
 892                 {'id': 'string23', '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}
 893         """
 894         if "_admin" not in indata:
 895             indata["_admin"] = {}
 896
 897         if "nsdOperationalState" in indata:
 898             if indata["nsdOperationalState"] in ("ENABLED", "DISABLED"):
 899                 indata["_admin"]["operationalState"] = indata.pop("nsdOperationalState")
 900             else:
 901                 raise EngineException("State '{}' is not a valid operational state"
 902                                       .format(indata["nsdOperationalState"]),
 903                                       http_code=HTTPStatus.BAD_REQUEST)
 904
 905         # In the case of user defined data, we need to put the data in the root of the object
 906         # to preserve current expected behaviour
 907         if "userDefinedData" in indata:
 908             data = indata.pop("userDefinedData")
 909             if type(data) == dict:
 910                 indata["_admin"]["userDefinedData"] = data
 911             else:
 912                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 913                                       .format(type(data)),
 914                                       http_code=HTTPStatus.BAD_REQUEST)
 915         if ("operationalState" in indata["_admin"] and
 916                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 917             raise EngineException("nsdOperationalState already {}".format(content["_admin"]["operationalState"]),
 918                                   http_code=HTTPStatus.CONFLICT)
 919         return indata
 920
 921     def _check_descriptor_dependencies(self, session, descriptor):
 922         """
 923         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 924         connection points are ok
 925         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 926         :param descriptor: descriptor to be inserted or edit
 927         :return: None or raises exception
 928         """
 929         if session["force"]:
 930             return
 931         vnfds_index = self._get_descriptor_constituent_vnfds_index(session, descriptor)
 932
 933         # Cross references validation in the descriptor and vnfd connection point validation
 934         for df in get_iterable(descriptor.get("df")):
 935             self.validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index)
 936
 937     def _get_descriptor_constituent_vnfds_index(self, session, descriptor):
 938         vnfds_index = {}
 939         if descriptor.get("vnfd-id") and not session["force"]:
 940             for vnfd_id in get_iterable(descriptor.get("vnfd-id")):
 941                 query_filter = self._get_project_filter(session)
 942                 query_filter["id"] = vnfd_id
 943                 vnf_list = self.db.get_list("vnfds", query_filter)
 944                 if not vnf_list:
 945                     raise EngineException("Descriptor error at 'vnfd-id'='{}' references a non "
 946                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 947                 vnfds_index[vnfd_id] = vnf_list[0]
 948         return vnfds_index
 949
 950     @staticmethod
 951     def validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index):
 952         for vnf_profile in get_iterable(df.get("vnf-profile")):
 953             vnfd = vnfds_index.get(vnf_profile["vnfd-id"])
 954             all_vnfd_ext_cpds = set()
 955             for ext_cpd in get_iterable(vnfd.get("ext-cpd")):
 956                 if ext_cpd.get('id'):
 957                     all_vnfd_ext_cpds.add(ext_cpd.get('id'))
 958
 959             for virtual_link in get_iterable(vnf_profile.get("virtual-link-connectivity")):
 960                 for vl_cpd in get_iterable(virtual_link.get("constituent-cpd-id")):
 961                     vl_cpd_id = vl_cpd.get('constituent-cpd-id')
 962                     if vl_cpd_id and vl_cpd_id not in all_vnfd_ext_cpds:
 963                         raise EngineException("Error at df[id='{}']:vnf-profile[id='{}']:virtual-link-connectivity"
 964                                               "[virtual-link-profile-id='{}']:constituent-cpd-id='{}' references a "
 965                                               "non existing ext-cpd:id inside vnfd '{}'"
 966                                               .format(df["id"], vnf_profile["id"],
 967                                                       virtual_link["virtual-link-profile-id"], vl_cpd_id, vnfd["id"]),
 968                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 969
 970     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 971         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 972
 973         self._check_descriptor_dependencies(session, final_content)
 974
 975     def check_conflict_on_del(self, session, _id, db_content):
 976         """
 977         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 978         that NSD can be public and be used by other projects.
 979         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 980         :param _id: nsd internal id
 981         :param db_content: The database content of the _id
 982         :return: None or raises EngineException with the conflict
 983         """
 984         if session["force"]:
 985             return
 986         descriptor = db_content
 987         descriptor_id = descriptor.get("id")
 988         if not descriptor_id:  # empty nsd not uploaded
 989             return
 990
 991         # check NSD used by NS
 992         _filter = self._get_project_filter(session)
 993         _filter["nsd-id"] = _id
 994         if self.db.get_list("nsrs", _filter):
 995             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
 996
 997         # check NSD referenced by NST
 998         del _filter["nsd-id"]
 999         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
1000         if self.db.get_list("nsts", _filter):
1001             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
1002                                   http_code=HTTPStatus.CONFLICT)
1003
1004     def sol005_projection(self, data):
1005         data["nsdOnboardingState"] = data["_admin"]["onboardingState"]
1006         data["nsdOperationalState"] = data["_admin"]["operationalState"]
1007         data["nsdUsageState"] = data["_admin"]["usageState"]
1008
1009         links = {}
1010         links["self"] = {"href": "/nsd/v1/ns_descriptors/{}".format(data["_id"])}
1011         links["nsd_content"] = {"href": "/nsd/v1/ns_descriptors/{}/nsd_content".format(data["_id"])}
1012         data["_links"] = links
1013
1014         return super().sol005_projection(data)
1015
1016
1017 class NstTopic(DescriptorTopic):
1018     topic = "nsts"
1019     topic_msg = "nst"
1020     quota_name = "slice_templates"
1021
1022     def __init__(self, db, fs, msg, auth):
1023         DescriptorTopic.__init__(self, db, fs, msg, auth)
1024
1025     def pyangbind_validation(self, item, data, force=False):
1026         try:
1027             mynst = nst_im()
1028             pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
1029                                              path_helper=True, skip_unknown=force)
1030             out = pybindJSON.dumps(mynst, mode="ietf")
1031             desc_out = self._remove_envelop(yaml.safe_load(out))
1032             return desc_out
1033         except Exception as e:
1034             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
1035                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
1036
1037     @staticmethod
1038     def _remove_envelop(indata=None):
1039         if not indata:
1040             return {}
1041         clean_indata = indata
1042
1043         if clean_indata.get('nst'):
1044             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
1045                 raise EngineException("'nst' must be a list only one element")
1046             clean_indata = clean_indata['nst'][0]
1047         elif clean_indata.get('nst:nst'):
1048             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
1049                 raise EngineException("'nst:nst' must be a list only one element")
1050             clean_indata = clean_indata['nst:nst'][0]
1051         return clean_indata
1052
1053     def _validate_input_new(self, indata, storage_params, force=False):
1054         indata.pop("onboardingState", None)
1055         indata.pop("operationalState", None)
1056         indata.pop("usageState", None)
1057         indata = self.pyangbind_validation("nsts", indata, force)
1058         return indata.copy()
1059
1060     def _check_descriptor_dependencies(self, session, descriptor):
1061         """
1062         Check that the dependent descriptors exist on a new descriptor or edition
1063         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1064         :param descriptor: descriptor to be inserted or edit
1065         :return: None or raises exception
1066         """
1067         if not descriptor.get("netslice-subnet"):
1068             return
1069         for nsd in descriptor["netslice-subnet"]:
1070             nsd_id = nsd["nsd-ref"]
1071             filter_q = self._get_project_filter(session)
1072             filter_q["id"] = nsd_id
1073             if not self.db.get_list("nsds", filter_q):
1074                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
1075                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
1076
1077     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
1078         super().check_conflict_on_edit(session, final_content, edit_content, _id)
1079
1080         self._check_descriptor_dependencies(session, final_content)
1081
1082     def check_conflict_on_del(self, session, _id, db_content):
1083         """
1084         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
1085         that NST can be public and be used by other projects.
1086         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1087         :param _id: nst internal id
1088         :param db_content: The database content of the _id.
1089         :return: None or raises EngineException with the conflict
1090         """
1091         # TODO: Check this method
1092         if session["force"]:
1093             return
1094         # Get Network Slice Template from Database
1095         _filter = self._get_project_filter(session)
1096         _filter["_admin.nst-id"] = _id
1097         if self.db.get_list("nsis", _filter):
1098             raise EngineException("there is at least one Netslice Instance using this descriptor",
1099                                   http_code=HTTPStatus.CONFLICT)
1100
1101     def sol005_projection(self, data):
1102         data["onboardingState"] = data["_admin"]["onboardingState"]
1103         data["operationalState"] = data["_admin"]["operationalState"]
1104         data["usageState"] = data["_admin"]["usageState"]
1105
1106         links = {}
1107         links["self"] = {"href": "/nst/v1/netslice_templates/{}".format(data["_id"])}
1108         links["nst"] = {"href": "/nst/v1/netslice_templates/{}/nst".format(data["_id"])}
1109         data["_links"] = links
1110
1111         return super().sol005_projection(data)
1112
1113
1114 class PduTopic(BaseTopic):
1115     topic = "pdus"
1116     topic_msg = "pdu"
1117     quota_name = "pduds"
1118     schema_new = pdu_new_schema
1119     schema_edit = pdu_edit_schema
1120
1121     def __init__(self, db, fs, msg, auth):
1122         BaseTopic.__init__(self, db, fs, msg, auth)
1123
1124     @staticmethod
1125     def format_on_new(content, project_id=None, make_public=False):
1126         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
1127         content["_admin"]["onboardingState"] = "CREATED"
1128         content["_admin"]["operationalState"] = "ENABLED"
1129         content["_admin"]["usageState"] = "NOT_IN_USE"
1130
1131     def check_conflict_on_del(self, session, _id, db_content):
1132         """
1133         Check that there is not any vnfr that uses this PDU
1134         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1135         :param _id: pdu internal id
1136         :param db_content: The database content of the _id.
1137         :return: None or raises EngineException with the conflict
1138         """
1139         if session["force"]:
1140             return
1141
1142         _filter = self._get_project_filter(session)
1143         _filter["vdur.pdu-id"] = _id
1144         if self.db.get_list("vnfrs", _filter):
1145             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)
1146
1147
1148 class VnfPkgOpTopic(BaseTopic):
1149     topic = "vnfpkgops"
1150     topic_msg = "vnfd"
1151     schema_new = vnfpkgop_new_schema
1152     schema_edit = None
1153
1154     def __init__(self, db, fs, msg, auth):
1155         BaseTopic.__init__(self, db, fs, msg, auth)
1156
1157     def edit(self, session, _id, indata=None, kwargs=None, content=None):
1158         raise EngineException("Method 'edit' not allowed for topic '{}'".format(self.topic),
1159                               HTTPStatus.METHOD_NOT_ALLOWED)
1160
1161     def delete(self, session, _id, dry_run=False):
1162         raise EngineException("Method 'delete' not allowed for topic '{}'".format(self.topic),
1163                               HTTPStatus.METHOD_NOT_ALLOWED)
1164
1165     def delete_list(self, session, filter_q=None):
1166         raise EngineException("Method 'delete_list' not allowed for topic '{}'".format(self.topic),
1167                               HTTPStatus.METHOD_NOT_ALLOWED)
1168
1169     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
1170         """
1171         Creates a new entry into database.
1172         :param rollback: list to append created items at database in case a rollback may to be done
1173         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1174         :param indata: data to be inserted
1175         :param kwargs: used to override the indata descriptor
1176         :param headers: http request headers
1177         :return: _id, op_id:
1178             _id: identity of the inserted data.
1179              op_id: None
1180         """
1181         self._update_input_with_kwargs(indata, kwargs)
1182         validate_input(indata, self.schema_new)
1183         vnfpkg_id = indata["vnfPkgId"]
1184         filter_q = BaseTopic._get_project_filter(session)
1185         filter_q["_id"] = vnfpkg_id
1186         vnfd = self.db.get_one("vnfds", filter_q)
1187         operation = indata["lcmOperationType"]
1188         kdu_name = indata["kdu_name"]
1189         for kdu in vnfd.get("kdu", []):
1190             if kdu["name"] == kdu_name:
1191                 helm_chart = kdu.get("helm-chart")
1192                 juju_bundle = kdu.get("juju-bundle")
1193                 break
1194         else:
1195             raise EngineException("Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id, kdu_name))
1196         if helm_chart:
1197             indata["helm-chart"] = helm_chart
1198             match = fullmatch(r"([^/]*)/([^/]*)", helm_chart)
1199             repo_name = match.group(1) if match else None
1200         elif juju_bundle:
1201             indata["juju-bundle"] = juju_bundle
1202             match = fullmatch(r"([^/]*)/([^/]*)", juju_bundle)
1203             repo_name = match.group(1) if match else None
1204         else:
1205             raise EngineException("Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']"
1206                                   .format(vnfpkg_id, kdu_name))
1207         if repo_name:
1208             del filter_q["_id"]
1209             filter_q["name"] = repo_name
1210             repo = self.db.get_one("k8srepos", filter_q)
1211             k8srepo_id = repo.get("_id")
1212             k8srepo_url = repo.get("url")
1213         else:
1214             k8srepo_id = None
1215             k8srepo_url = None
1216         indata["k8srepoId"] = k8srepo_id
1217         indata["k8srepo_url"] = k8srepo_url
1218         vnfpkgop_id = str(uuid4())
1219         vnfpkgop_desc = {
1220             "_id": vnfpkgop_id,
1221             "operationState": "PROCESSING",
1222             "vnfPkgId": vnfpkg_id,
1223             "lcmOperationType": operation,
1224             "isAutomaticInvocation": False,
1225             "isCancelPending": False,
1226             "operationParams": indata,
1227             "links": {
1228                 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id,
1229                 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id,
1230             }
1231         }
1232         self.format_on_new(vnfpkgop_desc, session["project_id"], make_public=session["public"])
1233         ctime = vnfpkgop_desc["_admin"]["created"]
1234         vnfpkgop_desc["statusEnteredTime"] = ctime
1235         vnfpkgop_desc["startTime"] = ctime
1236         self.db.create(self.topic, vnfpkgop_desc)
1237         rollback.append({"topic": self.topic, "_id": vnfpkgop_id})
1238         self.msg.write(self.topic_msg, operation, vnfpkgop_desc)
1239         return vnfpkgop_id, None