osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from time import time
  24 from uuid import uuid4
  25 from re import fullmatch
  26 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema, \
  27     validate_input, vnfpkgop_new_schema
  28 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  29 from osm_im.vnfd import vnfd as vnfd_im
  30 from osm_im.nsd import nsd as nsd_im
  31 from osm_im.nst import nst as nst_im
  32 from pyangbind.lib.serialise import pybindJSONDecoder
  33 import pyangbind.lib.pybindJSON as pybindJSON
  34
  35 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  36
  37
  38 class DescriptorTopic(BaseTopic):
  39
  40     def __init__(self, db, fs, msg, auth):
  41         BaseTopic.__init__(self, db, fs, msg, auth)
  42
  43     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  44         super().check_conflict_on_edit(session, final_content, edit_content, _id)
  45
  46         def _check_unique_id_name(descriptor, position=""):
  47             for desc_key, desc_item in descriptor.items():
  48                 if isinstance(desc_item, list) and desc_item:
  49                     used_ids = []
  50                     desc_item_id = None
  51                     for index, list_item in enumerate(desc_item):
  52                         if isinstance(list_item, dict):
  53                             _check_unique_id_name(list_item, "{}.{}[{}]"
  54                                                   .format(position, desc_key, index))
  55                             # Base case
  56                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  57                                 desc_item_id = "id" if list_item.get("id") else "name"
  58                             if desc_item_id and list_item.get(desc_item_id):
  59                                 if list_item[desc_item_id] in used_ids:
  60                                     position = "{}.{}[{}]".format(position, desc_key, index)
  61                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  62                                                           .format(desc_item_id, list_item[desc_item_id],
  63                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  64                                 used_ids.append(list_item[desc_item_id])
  65         _check_unique_id_name(final_content)
  66         # 1. validate again with pyangbind
  67         # 1.1. remove internal keys
  68         internal_keys = {}
  69         for k in ("_id", "_admin"):
  70             if k in final_content:
  71                 internal_keys[k] = final_content.pop(k)
  72         storage_params = internal_keys["_admin"].get("storage")
  73         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  74         # 1.2. modify final_content with a serialized version
  75         final_content.clear()
  76         final_content.update(serialized)
  77         # 1.3. restore internal keys
  78         for k, v in internal_keys.items():
  79             final_content[k] = v
  80
  81         if session["force"]:
  82             return
  83         # 2. check that this id is not present
  84         if "id" in edit_content:
  85             _filter = self._get_project_filter(session)
  86             _filter["id"] = final_content["id"]
  87             _filter["_id.neq"] = _id
  88             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  89                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  90                                                                                                final_content["id"]),
  91                                       HTTPStatus.CONFLICT)
  92
  93     @staticmethod
  94     def format_on_new(content, project_id=None, make_public=False):
  95         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  96         content["_admin"]["onboardingState"] = "CREATED"
  97         content["_admin"]["operationalState"] = "DISABLED"
  98         content["_admin"]["usageState"] = "NOT_IN_USE"
  99
 100     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 101         """
 102         Deletes file system storage associated with the descriptor
 103         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 104         :param _id: server internal id
 105         :param db_content: The database content of the descriptor
 106         :param not_send_msg: To not send message (False) or store content (list) instead
 107         :return: None if ok or raises EngineException with the problem
 108         """
 109         self.fs.file_delete(_id, ignore_non_exist=True)
 110         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 111
 112     @staticmethod
 113     def get_one_by_id(db, session, topic, id):
 114         # find owned by this project
 115         _filter = BaseTopic._get_project_filter(session)
 116         _filter["id"] = id
 117         desc_list = db.get_list(topic, _filter)
 118         if len(desc_list) == 1:
 119             return desc_list[0]
 120         elif len(desc_list) > 1:
 121             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 122                               HTTPStatus.CONFLICT)
 123
 124         # not found any: try to find public
 125         _filter = BaseTopic._get_project_filter(session)
 126         _filter["id"] = id
 127         desc_list = db.get_list(topic, _filter)
 128         if not desc_list:
 129             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 130         elif len(desc_list) == 1:
 131             return desc_list[0]
 132         else:
 133             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 134                 topic[:-1], id), HTTPStatus.CONFLICT)
 135
 136     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 137         """
 138         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 139         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 140         (self.upload_content)
 141         :param rollback: list to append created items at database in case a rollback may to be done
 142         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 143         :param indata: data to be inserted
 144         :param kwargs: used to override the indata descriptor
 145         :param headers: http request headers
 146         :return: _id, None: identity of the inserted data; and None as there is not any operation
 147         """
 148
 149         # No needed to capture exceptions
 150         # Check Quota
 151         self.check_quota(session)
 152
 153         # _remove_envelop
 154         if indata:
 155             if "userDefinedData" in indata:
 156                 indata = indata['userDefinedData']
 157
 158         # Override descriptor with query string kwargs
 159         self._update_input_with_kwargs(indata, kwargs)
 160         # uncomment when this method is implemented.
 161         # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 162         # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 163
 164         content = {"_admin": {"userDefinedData": indata}}
 165         self.format_on_new(content, session["project_id"], make_public=session["public"])
 166         _id = self.db.create(self.topic, content)
 167         rollback.append({"topic": self.topic, "_id": _id})
 168         self._send_msg("created", {"_id": _id})
 169         return _id, None
 170
 171     def upload_content(self, session, _id, indata, kwargs, headers):
 172         """
 173         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 174         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 175         :param _id : the nsd,vnfd is already created, this is the id
 176         :param indata: http body request
 177         :param kwargs: user query string to override parameters. NOT USED
 178         :param headers:  http request headers
 179         :return: True if package is completely uploaded or False if partial content has been uploded
 180             Raise exception on error
 181         """
 182         # Check that _id exists and it is valid
 183         current_desc = self.show(session, _id)
 184
 185         content_range_text = headers.get("Content-Range")
 186         expected_md5 = headers.get("Content-File-MD5")
 187         compressed = None
 188         content_type = headers.get("Content-Type")
 189         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 190                 "application/zip" in content_type:
 191             compressed = "gzip"
 192         filename = headers.get("Content-Filename")
 193         if not filename:
 194             filename = "package.tar.gz" if compressed else "package"
 195         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 196         file_pkg = None
 197         error_text = ""
 198         try:
 199             if content_range_text:
 200                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 201                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 202                     raise IndexError()
 203                 start = int(content_range[1])
 204                 end = int(content_range[2]) + 1
 205                 total = int(content_range[3])
 206             else:
 207                 start = 0
 208             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 209
 210             if start:
 211                 if not self.fs.file_exists(temp_folder, 'dir'):
 212                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 213             else:
 214                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 215                 self.fs.mkdir(temp_folder)
 216
 217             storage = self.fs.get_params()
 218             storage["folder"] = _id
 219
 220             file_path = (temp_folder, filename)
 221             if self.fs.file_exists(file_path, 'file'):
 222                 file_size = self.fs.file_size(file_path)
 223             else:
 224                 file_size = 0
 225             if file_size != start:
 226                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 227                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 228             file_pkg = self.fs.file_open(file_path, 'a+b')
 229             if isinstance(indata, dict):
 230                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 231                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 232             else:
 233                 indata_len = 0
 234                 while True:
 235                     indata_text = indata.read(4096)
 236                     indata_len += len(indata_text)
 237                     if not indata_text:
 238                         break
 239                     file_pkg.write(indata_text)
 240             if content_range_text:
 241                 if indata_len != end-start:
 242                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 243                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 244                 if end != total:
 245                     # TODO update to UPLOADING
 246                     return False
 247
 248             # PACKAGE UPLOADED
 249             if expected_md5:
 250                 file_pkg.seek(0, 0)
 251                 file_md5 = md5()
 252                 chunk_data = file_pkg.read(1024)
 253                 while chunk_data:
 254                     file_md5.update(chunk_data)
 255                     chunk_data = file_pkg.read(1024)
 256                 if expected_md5 != file_md5.hexdigest():
 257                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 258             file_pkg.seek(0, 0)
 259             if compressed == "gzip":
 260                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 261                 descriptor_file_name = None
 262                 for tarinfo in tar:
 263                     tarname = tarinfo.name
 264                     tarname_path = tarname.split("/")
 265                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 266                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 267                     if len(tarname_path) == 1 and not tarinfo.isdir():
 268                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 269                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 270                         storage["pkg-dir"] = tarname_path[0]
 271                         if len(tarname_path) == 2:
 272                             if descriptor_file_name:
 273                                 raise EngineException(
 274                                     "Found more than one descriptor file at package descriptor tar.gz")
 275                             descriptor_file_name = tarname
 276                 if not descriptor_file_name:
 277                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 278                 storage["descriptor"] = descriptor_file_name
 279                 storage["zipfile"] = filename
 280                 self.fs.file_extract(tar, temp_folder)
 281                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 282                     content = descriptor_file.read()
 283             else:
 284                 content = file_pkg.read()
 285                 storage["descriptor"] = descriptor_file_name = filename
 286
 287             if descriptor_file_name.endswith(".json"):
 288                 error_text = "Invalid json format "
 289                 indata = json.load(content)
 290             else:
 291                 error_text = "Invalid yaml format "
 292                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 293
 294             current_desc["_admin"]["storage"] = storage
 295             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 296             current_desc["_admin"]["operationalState"] = "ENABLED"
 297
 298             indata = self._remove_envelop(indata)
 299
 300             # Override descriptor with query string kwargs
 301             if kwargs:
 302                 self._update_input_with_kwargs(indata, kwargs)
 303             # it will call overrides method at VnfdTopic or NsdTopic
 304             # indata = self._validate_input_edit(indata, force=session["force"])
 305
 306             deep_update_rfc7396(current_desc, indata)
 307             self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 308             current_desc["_admin"]["modified"] = time()
 309             self.db.replace(self.topic, _id, current_desc)
 310             self.fs.dir_rename(temp_folder, _id)
 311
 312             indata["_id"] = _id
 313             self._send_msg("edited", indata)
 314
 315             # TODO if descriptor has changed because kwargs update content and remove cached zip
 316             # TODO if zip is not present creates one
 317             return True
 318
 319         except EngineException:
 320             raise
 321         except IndexError:
 322             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 323                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 324         except IOError as e:
 325             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 326         except tarfile.ReadError as e:
 327             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 328         except (ValueError, yaml.YAMLError) as e:
 329             raise EngineException(error_text + str(e))
 330         except ValidationError as e:
 331             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 332         finally:
 333             if file_pkg:
 334                 file_pkg.close()
 335
 336     def get_file(self, session, _id, path=None, accept_header=None):
 337         """
 338         Return the file content of a vnfd or nsd
 339         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 340         :param _id: Identity of the vnfd, nsd
 341         :param path: artifact path or "$DESCRIPTOR" or None
 342         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 343         :return: opened file plus Accept format or raises an exception
 344         """
 345         accept_text = accept_zip = False
 346         if accept_header:
 347             if 'text/plain' in accept_header or '*/*' in accept_header:
 348                 accept_text = True
 349             if 'application/zip' in accept_header or '*/*' in accept_header:
 350                 accept_zip = 'application/zip'
 351             elif 'application/gzip' in accept_header:
 352                 accept_zip = 'application/gzip'
 353
 354         if not accept_text and not accept_zip:
 355             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 356                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 357
 358         content = self.show(session, _id)
 359         if content["_admin"]["onboardingState"] != "ONBOARDED":
 360             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 361                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 362                                   http_code=HTTPStatus.CONFLICT)
 363         storage = content["_admin"]["storage"]
 364         if path is not None and path != "$DESCRIPTOR":   # artifacts
 365             if not storage.get('pkg-dir'):
 366                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 367             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 368                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 369                 return folder_content, "text/plain"
 370                 # TODO manage folders in http
 371             else:
 372                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 373                     "application/octet-stream"
 374
 375         # pkgtype   accept  ZIP  TEXT    -> result
 376         # manyfiles         yes  X       -> zip
 377         #                   no   yes     -> error
 378         # onefile           yes  no      -> zip
 379         #                   X    yes     -> text
 380
 381         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 382             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 383         elif storage.get('pkg-dir') and not accept_zip:
 384             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 385                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 386         else:
 387             if not storage.get('zipfile'):
 388                 # TODO generate zipfile if not present
 389                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 390                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 391             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 392
 393     def pyangbind_validation(self, item, data, force=False):
 394         try:
 395             if item == "vnfds":
 396                 myvnfd = vnfd_im()
 397                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 398                                                  path_helper=True, skip_unknown=force)
 399                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 400             elif item == "nsds":
 401                 mynsd = nsd_im()
 402                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 403                                                  path_helper=True, skip_unknown=force)
 404                 out = pybindJSON.dumps(mynsd, mode="ietf")
 405             elif item == "nsts":
 406                 mynst = nst_im()
 407                 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
 408                                                  path_helper=True, skip_unknown=force)
 409                 out = pybindJSON.dumps(mynst, mode="ietf")
 410             else:
 411                 raise EngineException("Not possible to validate '{}' item".format(item),
 412                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 413
 414             desc_out = self._remove_envelop(yaml.safe_load(out))
 415             return desc_out
 416
 417         except Exception as e:
 418             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 419                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 420
 421     def _validate_input_edit(self, indata, content, force=False):
 422         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 423         if "_id" in indata:
 424             indata.pop("_id")
 425         if "_admin" not in indata:
 426             indata["_admin"] = {}
 427
 428         if "operationalState" in indata:
 429             if indata["operationalState"] in ("ENABLED", "DISABLED"):
 430                 indata["_admin"]["operationalState"] = indata.pop("operationalState")
 431             else:
 432                 raise EngineException("State '{}' is not a valid operational state"
 433                                       .format(indata["operationalState"]),
 434                                       http_code=HTTPStatus.BAD_REQUEST)
 435
 436         # In the case of user defined data, we need to put the data in the root of the object
 437         # to preserve current expected behaviour
 438         if "userDefinedData" in indata:
 439             data = indata.pop("userDefinedData")
 440             if type(data) == dict:
 441                 indata["_admin"]["userDefinedData"] = data
 442             else:
 443                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 444                                       .format(type(data)),
 445                                       http_code=HTTPStatus.BAD_REQUEST)
 446
 447         if ("operationalState" in indata["_admin"] and
 448                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 449             raise EngineException("operationalState already {}".format(content["_admin"]["operationalState"]),
 450                                   http_code=HTTPStatus.CONFLICT)
 451
 452         return indata
 453
 454
 455 class VnfdTopic(DescriptorTopic):
 456     topic = "vnfds"
 457     topic_msg = "vnfd"
 458
 459     def __init__(self, db, fs, msg, auth):
 460         DescriptorTopic.__init__(self, db, fs, msg, auth)
 461
 462     @staticmethod
 463     def _remove_envelop(indata=None):
 464         if not indata:
 465             return {}
 466         clean_indata = indata
 467         if clean_indata.get('vnfd:vnfd-catalog'):
 468             clean_indata = clean_indata['vnfd:vnfd-catalog']
 469         elif clean_indata.get('vnfd-catalog'):
 470             clean_indata = clean_indata['vnfd-catalog']
 471         if clean_indata.get('vnfd'):
 472             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 473                 raise EngineException("'vnfd' must be a list of only one element")
 474             clean_indata = clean_indata['vnfd'][0]
 475         elif clean_indata.get('vnfd:vnfd'):
 476             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 477                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 478             clean_indata = clean_indata['vnfd:vnfd'][0]
 479         return clean_indata
 480
 481     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 482         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 483
 484         # set type of vnfd
 485         contains_pdu = False
 486         contains_vdu = False
 487         for vdu in get_iterable(final_content.get("vdu")):
 488             if vdu.get("pdu-type"):
 489                 contains_pdu = True
 490             else:
 491                 contains_vdu = True
 492         if contains_pdu:
 493             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 494         elif contains_vdu:
 495             final_content["_admin"]["type"] = "vnfd"
 496         # if neither vud nor pdu do not fill type
 497
 498     def check_conflict_on_del(self, session, _id, db_content):
 499         """
 500         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 501         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 502         that uses this vnfd
 503         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 504         :param _id: vnfd internal id
 505         :param db_content: The database content of the _id.
 506         :return: None or raises EngineException with the conflict
 507         """
 508         if session["force"]:
 509             return
 510         descriptor = db_content
 511         descriptor_id = descriptor.get("id")
 512         if not descriptor_id:  # empty vnfd not uploaded
 513             return
 514
 515         _filter = self._get_project_filter(session)
 516
 517         # check vnfrs using this vnfd
 518         _filter["vnfd-id"] = _id
 519         if self.db.get_list("vnfrs", _filter):
 520             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 521
 522         # check NSD referencing this VNFD
 523         del _filter["vnfd-id"]
 524         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 525         if self.db.get_list("nsds", _filter):
 526             raise EngineException("There is at least one NSD referencing this descriptor",
 527                                   http_code=HTTPStatus.CONFLICT)
 528
 529     def _validate_input_new(self, indata, storage_params, force=False):
 530         indata.pop("onboardingState", None)
 531         indata.pop("operationalState", None)
 532         indata.pop("usageState", None)
 533
 534         indata.pop("links", None)
 535
 536         indata = self.pyangbind_validation("vnfds", indata, force)
 537         # Cross references validation in the descriptor
 538         if indata.get("vdu"):
 539             if not indata.get("mgmt-interface"):
 540                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 541                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 542             if indata["mgmt-interface"].get("cp"):
 543                 for cp in get_iterable(indata.get("connection-point")):
 544                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 545                         break
 546                 else:
 547                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 548                                           .format(indata["mgmt-interface"]["cp"]),
 549                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 550
 551         for vdu in get_iterable(indata.get("vdu")):
 552             icp_refs = []
 553             ecp_refs = []
 554             for interface in get_iterable(vdu.get("interface")):
 555                 if interface.get("external-connection-point-ref"):
 556                     if interface.get("external-connection-point-ref") in ecp_refs:
 557                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 558                                               "is referenced by other interface"
 559                                               .format(vdu["id"], interface["name"],
 560                                                       interface["external-connection-point-ref"]),
 561                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 562                     ecp_refs.append(interface.get("external-connection-point-ref"))
 563                     for cp in get_iterable(indata.get("connection-point")):
 564                         if cp["name"] == interface["external-connection-point-ref"]:
 565                             break
 566                     else:
 567                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 568                                               "must match an existing connection-point"
 569                                               .format(vdu["id"], interface["name"],
 570                                                       interface["external-connection-point-ref"]),
 571                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 572                 elif interface.get("internal-connection-point-ref"):
 573                     if interface.get("internal-connection-point-ref") in icp_refs:
 574                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 575                                               "is referenced by other interface"
 576                                               .format(vdu["id"], interface["name"],
 577                                                       interface["internal-connection-point-ref"]),
 578                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 579                     icp_refs.append(interface.get("internal-connection-point-ref"))
 580                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 581                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 582                             break
 583                     else:
 584                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 585                                               "must match an existing vdu:internal-connection-point"
 586                                               .format(vdu["id"], interface["name"],
 587                                                       interface["internal-connection-point-ref"]),
 588                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 589             # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 590             if vdu.get("vdu-configuration"):
 591                 if vdu["vdu-configuration"].get("juju"):
 592                     if not self._validate_package_folders(storage_params, 'charms'):
 593                         raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
 594                                               "package".format(indata["id"], vdu["id"]))
 595             # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
 596             if vdu.get("cloud-init-file"):
 597                 if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 598                     raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 599                                           "package".format(indata["id"], vdu["id"]))
 600         # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 601         if indata.get("vnf-configuration"):
 602             if indata["vnf-configuration"].get("juju"):
 603                 if not self._validate_package_folders(storage_params, 'charms'):
 604                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 605                                           "package".format(indata["id"]))
 606         vld_names = []  # For detection of duplicated VLD names
 607         for ivld in get_iterable(indata.get("internal-vld")):
 608             # BEGIN Detection of duplicated VLD names
 609             ivld_name = ivld.get("name")
 610             if ivld_name:
 611                 if ivld_name in vld_names:
 612                     raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
 613                                           .format(ivld["name"], indata["id"], ivld["id"]),
 614                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 615                 else:
 616                     vld_names.append(ivld_name)
 617             # END Detection of duplicated VLD names
 618             for icp in get_iterable(ivld.get("internal-connection-point")):
 619                 icp_mark = False
 620                 for vdu in get_iterable(indata.get("vdu")):
 621                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 622                         if icp["id-ref"] == internal_cp["id"]:
 623                             icp_mark = True
 624                             break
 625                     if icp_mark:
 626                         break
 627                 else:
 628                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 629                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 630                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 631             if ivld.get("ip-profile-ref"):
 632                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 633                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 634                         break
 635                 else:
 636                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 637                         ivld["id"], ivld["ip-profile-ref"]),
 638                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 639         for mp in get_iterable(indata.get("monitoring-param")):
 640             if mp.get("vdu-monitoring-param"):
 641                 mp_vmp_mark = False
 642                 for vdu in get_iterable(indata.get("vdu")):
 643                     for vmp in get_iterable(vdu.get("monitoring-param")):
 644                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 645                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 646                             mp_vmp_mark = True
 647                             break
 648                     if mp_vmp_mark:
 649                         break
 650                 else:
 651                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 652                                           "defined at vdu[id='{}'] or vdu does not exist"
 653                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 654                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 655                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 656             elif mp.get("vdu-metric"):
 657                 mp_vm_mark = False
 658                 for vdu in get_iterable(indata.get("vdu")):
 659                     if vdu.get("vdu-configuration"):
 660                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 661                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 662                                     mp["vdu-metric"]["vdu-ref"]:
 663                                 mp_vm_mark = True
 664                                 break
 665                         if mp_vm_mark:
 666                             break
 667                 else:
 668                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 669                                           "vdu[id='{}'] or vdu does not exist"
 670                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 671                                                   mp["vdu-metric"]["vdu-ref"]),
 672                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 673
 674         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 675             for sp in get_iterable(sgd.get("scaling-policy")):
 676                 for sc in get_iterable(sp.get("scaling-criteria")):
 677                     for mp in get_iterable(indata.get("monitoring-param")):
 678                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 679                             break
 680                     else:
 681                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 682                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 683                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 684                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 685             for sgd_vdu in get_iterable(sgd.get("vdu")):
 686                 sgd_vdu_mark = False
 687                 for vdu in get_iterable(indata.get("vdu")):
 688                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 689                         sgd_vdu_mark = True
 690                         break
 691                 if sgd_vdu_mark:
 692                     break
 693             else:
 694                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 695                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 696                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 697             for sca in get_iterable(sgd.get("scaling-config-action")):
 698                 if not indata.get("vnf-configuration"):
 699                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 700                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 701                                           .format(sgd["name"]),
 702                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 703                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 704                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 705                         break
 706                 else:
 707                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 708                                           "primitive-name-ref='{}' does not match any "
 709                                           "vnf-configuration:config-primitive:name"
 710                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 711                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 712         return indata
 713
 714     def _validate_package_folders(self, storage_params, folder, file=None):
 715         if not storage_params or not storage_params.get("pkg-dir"):
 716             return False
 717         else:
 718             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 719                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 720             else:
 721                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 722             if file:
 723                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 724             else:
 725                 if self.fs.file_exists(f, 'dir'):
 726                     if self.fs.dir_ls(f):
 727                         return True
 728             return False
 729
 730     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 731         """
 732         Deletes associate file system storage (via super)
 733         Deletes associated vnfpkgops from database.
 734         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 735         :param _id: server internal id
 736         :param db_content: The database content of the descriptor
 737         :return: None
 738         :raises: FsException in case of error while deleting associated storage
 739         """
 740         super().delete_extra(session, _id, db_content, not_send_msg)
 741         self.db.del_list("vnfpkgops", {"vnfPkgId": _id})
 742
 743     def sol005_projection(self, data):
 744         data["onboardingState"] = data["_admin"]["onboardingState"]
 745         data["operationalState"] = data["_admin"]["operationalState"]
 746         data["usageState"] = data["_admin"]["usageState"]
 747
 748         links = {}
 749         links["self"] = {"href": "/vnfpkgm/v1/vnf_packages/{}".format(data["_id"])}
 750         links["vnfd"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(data["_id"])}
 751         links["packageContent"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/package_content".format(data["_id"])}
 752         data["_links"] = links
 753
 754         return super().sol005_projection(data)
 755
 756
 757 class NsdTopic(DescriptorTopic):
 758     topic = "nsds"
 759     topic_msg = "nsd"
 760
 761     def __init__(self, db, fs, msg, auth):
 762         DescriptorTopic.__init__(self, db, fs, msg, auth)
 763
 764     @staticmethod
 765     def _remove_envelop(indata=None):
 766         if not indata:
 767             return {}
 768         clean_indata = indata
 769
 770         if clean_indata.get('nsd:nsd-catalog'):
 771             clean_indata = clean_indata['nsd:nsd-catalog']
 772         elif clean_indata.get('nsd-catalog'):
 773             clean_indata = clean_indata['nsd-catalog']
 774         if clean_indata.get('nsd'):
 775             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 776                 raise EngineException("'nsd' must be a list of only one element")
 777             clean_indata = clean_indata['nsd'][0]
 778         elif clean_indata.get('nsd:nsd'):
 779             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 780                 raise EngineException("'nsd:nsd' must be a list of only one element")
 781             clean_indata = clean_indata['nsd:nsd'][0]
 782         return clean_indata
 783
 784     def _validate_input_new(self, indata, storage_params, force=False):
 785         indata.pop("nsdOnboardingState", None)
 786         indata.pop("nsdOperationalState", None)
 787         indata.pop("nsdUsageState", None)
 788
 789         indata.pop("links", None)
 790
 791         indata = self.pyangbind_validation("nsds", indata, force)
 792         # Cross references validation in the descriptor
 793         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 794         for vld in get_iterable(indata.get("vld")):
 795             if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
 796                 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
 797                                       " You cannot set an ip-profile when mgmt-network is True"
 798                                       .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 799             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 800                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 801                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 802                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 803                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 804                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 805                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 806                                                                  constituent_vnfd["member-vnf-index"],
 807                                                                  constituent_vnfd["vnfd-id-ref"]),
 808                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 809                         break
 810                 else:
 811                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 812                                           "does not match any constituent-vnfd:member-vnf-index"
 813                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 814                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 815         # Check VNFFGD
 816         for fgd in get_iterable(indata.get("vnffgd")):
 817             for cls in get_iterable(fgd.get("classifier")):
 818                 rspref = cls.get("rsp-id-ref")
 819                 for rsp in get_iterable(fgd.get("rsp")):
 820                     rspid = rsp.get("id")
 821                     if rspid and rspref and rspid == rspref:
 822                         break
 823                 else:
 824                     raise EngineException(
 825                         "Error at vnffgd[id='{}']:classifier[id='{}']:rsp-id-ref '{}' does not match any rsp:id"
 826                         .format(fgd["id"], cls["id"], rspref),
 827                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 828         return indata
 829
 830     def _validate_input_edit(self, indata, content, force=False):
 831         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 832         """
 833         indata looks as follows:
 834             - In the new case (conformant)
 835                 {'nsdOperationalState': 'DISABLED', 'userDefinedData': {'id': 'string23',
 836                 '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}}
 837             - In the old case (backwards-compatible)
 838                 {'id': 'string23', '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}
 839         """
 840         if "_admin" not in indata:
 841             indata["_admin"] = {}
 842
 843         if "nsdOperationalState" in indata:
 844             if indata["nsdOperationalState"] in ("ENABLED", "DISABLED"):
 845                 indata["_admin"]["operationalState"] = indata.pop("nsdOperationalState")
 846             else:
 847                 raise EngineException("State '{}' is not a valid operational state"
 848                                       .format(indata["nsdOperationalState"]),
 849                                       http_code=HTTPStatus.BAD_REQUEST)
 850
 851         # In the case of user defined data, we need to put the data in the root of the object
 852         # to preserve current expected behaviour
 853         if "userDefinedData" in indata:
 854             data = indata.pop("userDefinedData")
 855             if type(data) == dict:
 856                 indata["_admin"]["userDefinedData"] = data
 857             else:
 858                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 859                                       .format(type(data)),
 860                                       http_code=HTTPStatus.BAD_REQUEST)
 861         if ("operationalState" in indata["_admin"] and
 862                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 863             raise EngineException("nsdOperationalState already {}".format(content["_admin"]["operationalState"]),
 864                                   http_code=HTTPStatus.CONFLICT)
 865         return indata
 866
 867     def _check_descriptor_dependencies(self, session, descriptor):
 868         """
 869         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 870         connection points are ok
 871         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 872         :param descriptor: descriptor to be inserted or edit
 873         :return: None or raises exception
 874         """
 875         if session["force"]:
 876             return
 877         member_vnfd_index = {}
 878         if descriptor.get("constituent-vnfd") and not session["force"]:
 879             for vnf in descriptor["constituent-vnfd"]:
 880                 vnfd_id = vnf["vnfd-id-ref"]
 881                 filter_q = self._get_project_filter(session)
 882                 filter_q["id"] = vnfd_id
 883                 vnf_list = self.db.get_list("vnfds", filter_q)
 884                 if not vnf_list:
 885                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 886                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 887                 # elif len(vnf_list) > 1:
 888                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 889                 #                           http_code=HTTPStatus.CONFLICT)
 890                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 891
 892         # Cross references validation in the descriptor and vnfd connection point validation
 893         for vld in get_iterable(descriptor.get("vld")):
 894             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 895                 # look if this vnfd contains this connection point
 896                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 897                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 898                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 899                         break
 900                 else:
 901                     raise EngineException(
 902                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 903                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 904                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 905                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 906                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 907
 908     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 909         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 910
 911         self._check_descriptor_dependencies(session, final_content)
 912
 913     def check_conflict_on_del(self, session, _id, db_content):
 914         """
 915         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 916         that NSD can be public and be used by other projects.
 917         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 918         :param _id: nsd internal id
 919         :param db_content: The database content of the _id
 920         :return: None or raises EngineException with the conflict
 921         """
 922         if session["force"]:
 923             return
 924         descriptor = db_content
 925         descriptor_id = descriptor.get("id")
 926         if not descriptor_id:  # empty nsd not uploaded
 927             return
 928
 929         # check NSD used by NS
 930         _filter = self._get_project_filter(session)
 931         _filter["nsd-id"] = _id
 932         if self.db.get_list("nsrs", _filter):
 933             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
 934
 935         # check NSD referenced by NST
 936         del _filter["nsd-id"]
 937         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
 938         if self.db.get_list("nsts", _filter):
 939             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
 940                                   http_code=HTTPStatus.CONFLICT)
 941
 942     def sol005_projection(self, data):
 943         data["nsdOnboardingState"] = data["_admin"]["onboardingState"]
 944         data["nsdOperationalState"] = data["_admin"]["operationalState"]
 945         data["nsdUsageState"] = data["_admin"]["usageState"]
 946
 947         links = {}
 948         links["self"] = {"href": "/nsd/v1/ns_descriptors/{}".format(data["_id"])}
 949         links["nsd_content"] = {"href": "/nsd/v1/ns_descriptors/{}/nsd_content".format(data["_id"])}
 950         data["_links"] = links
 951
 952         return super().sol005_projection(data)
 953
 954
 955 class NstTopic(DescriptorTopic):
 956     topic = "nsts"
 957     topic_msg = "nst"
 958     quota_name = "slice_templates"
 959
 960     def __init__(self, db, fs, msg, auth):
 961         DescriptorTopic.__init__(self, db, fs, msg, auth)
 962
 963     @staticmethod
 964     def _remove_envelop(indata=None):
 965         if not indata:
 966             return {}
 967         clean_indata = indata
 968
 969         if clean_indata.get('nst'):
 970             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 971                 raise EngineException("'nst' must be a list only one element")
 972             clean_indata = clean_indata['nst'][0]
 973         elif clean_indata.get('nst:nst'):
 974             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
 975                 raise EngineException("'nst:nst' must be a list only one element")
 976             clean_indata = clean_indata['nst:nst'][0]
 977         return clean_indata
 978
 979     def _validate_input_new(self, indata, storage_params, force=False):
 980         indata.pop("onboardingState", None)
 981         indata.pop("operationalState", None)
 982         indata.pop("usageState", None)
 983         indata = self.pyangbind_validation("nsts", indata, force)
 984         return indata.copy()
 985
 986     def _check_descriptor_dependencies(self, session, descriptor):
 987         """
 988         Check that the dependent descriptors exist on a new descriptor or edition
 989         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 990         :param descriptor: descriptor to be inserted or edit
 991         :return: None or raises exception
 992         """
 993         if not descriptor.get("netslice-subnet"):
 994             return
 995         for nsd in descriptor["netslice-subnet"]:
 996             nsd_id = nsd["nsd-ref"]
 997             filter_q = self._get_project_filter(session)
 998             filter_q["id"] = nsd_id
 999             if not self.db.get_list("nsds", filter_q):
1000                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
1001                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
1002
1003     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
1004         super().check_conflict_on_edit(session, final_content, edit_content, _id)
1005
1006         self._check_descriptor_dependencies(session, final_content)
1007
1008     def check_conflict_on_del(self, session, _id, db_content):
1009         """
1010         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
1011         that NST can be public and be used by other projects.
1012         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1013         :param _id: nst internal id
1014         :param db_content: The database content of the _id.
1015         :return: None or raises EngineException with the conflict
1016         """
1017         # TODO: Check this method
1018         if session["force"]:
1019             return
1020         # Get Network Slice Template from Database
1021         _filter = self._get_project_filter(session)
1022         _filter["_admin.nst-id"] = _id
1023         if self.db.get_list("nsis", _filter):
1024             raise EngineException("there is at least one Netslice Instance using this descriptor",
1025                                   http_code=HTTPStatus.CONFLICT)
1026
1027     def sol005_projection(self, data):
1028         data["onboardingState"] = data["_admin"]["onboardingState"]
1029         data["operationalState"] = data["_admin"]["operationalState"]
1030         data["usageState"] = data["_admin"]["usageState"]
1031
1032         links = {}
1033         links["self"] = {"href": "/nst/v1/netslice_templates/{}".format(data["_id"])}
1034         links["nst"] = {"href": "/nst/v1/netslice_templates/{}/nst".format(data["_id"])}
1035         data["_links"] = links
1036
1037         return super().sol005_projection(data)
1038
1039
1040 class PduTopic(BaseTopic):
1041     topic = "pdus"
1042     topic_msg = "pdu"
1043     quota_name = "pduds"
1044     schema_new = pdu_new_schema
1045     schema_edit = pdu_edit_schema
1046
1047     def __init__(self, db, fs, msg, auth):
1048         BaseTopic.__init__(self, db, fs, msg, auth)
1049
1050     @staticmethod
1051     def format_on_new(content, project_id=None, make_public=False):
1052         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
1053         content["_admin"]["onboardingState"] = "CREATED"
1054         content["_admin"]["operationalState"] = "ENABLED"
1055         content["_admin"]["usageState"] = "NOT_IN_USE"
1056
1057     def check_conflict_on_del(self, session, _id, db_content):
1058         """
1059         Check that there is not any vnfr that uses this PDU
1060         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1061         :param _id: pdu internal id
1062         :param db_content: The database content of the _id.
1063         :return: None or raises EngineException with the conflict
1064         """
1065         if session["force"]:
1066             return
1067
1068         _filter = self._get_project_filter(session)
1069         _filter["vdur.pdu-id"] = _id
1070         if self.db.get_list("vnfrs", _filter):
1071             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)
1072
1073
1074 class VnfPkgOpTopic(BaseTopic):
1075     topic = "vnfpkgops"
1076     topic_msg = "vnfd"
1077     schema_new = vnfpkgop_new_schema
1078     schema_edit = None
1079
1080     def __init__(self, db, fs, msg, auth):
1081         BaseTopic.__init__(self, db, fs, msg, auth)
1082
1083     def edit(self, session, _id, indata=None, kwargs=None, content=None):
1084         raise EngineException("Method 'edit' not allowed for topic '{}'".format(self.topic),
1085                               HTTPStatus.METHOD_NOT_ALLOWED)
1086
1087     def delete(self, session, _id, dry_run=False):
1088         raise EngineException("Method 'delete' not allowed for topic '{}'".format(self.topic),
1089                               HTTPStatus.METHOD_NOT_ALLOWED)
1090
1091     def delete_list(self, session, filter_q=None):
1092         raise EngineException("Method 'delete_list' not allowed for topic '{}'".format(self.topic),
1093                               HTTPStatus.METHOD_NOT_ALLOWED)
1094
1095     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
1096         """
1097         Creates a new entry into database.
1098         :param rollback: list to append created items at database in case a rollback may to be done
1099         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1100         :param indata: data to be inserted
1101         :param kwargs: used to override the indata descriptor
1102         :param headers: http request headers
1103         :return: _id, op_id:
1104             _id: identity of the inserted data.
1105              op_id: None
1106         """
1107         self._update_input_with_kwargs(indata, kwargs)
1108         validate_input(indata, self.schema_new)
1109         vnfpkg_id = indata["vnfPkgId"]
1110         filter_q = BaseTopic._get_project_filter(session)
1111         filter_q["_id"] = vnfpkg_id
1112         vnfd = self.db.get_one("vnfds", filter_q)
1113         operation = indata["lcmOperationType"]
1114         kdu_name = indata["kdu_name"]
1115         for kdu in vnfd.get("kdu", []):
1116             if kdu["name"] == kdu_name:
1117                 helm_chart = kdu.get("helm-chart")
1118                 juju_bundle = kdu.get("juju-bundle")
1119                 break
1120         else:
1121             raise EngineException("Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id, kdu_name))
1122         if helm_chart:
1123             indata["helm-chart"] = helm_chart
1124             match = fullmatch(r"([^/]*)/([^/]*)", helm_chart)
1125             repo_name = match.group(1) if match else None
1126         elif juju_bundle:
1127             indata["juju-bundle"] = juju_bundle
1128             match = fullmatch(r"([^/]*)/([^/]*)", juju_bundle)
1129             repo_name = match.group(1) if match else None
1130         else:
1131             raise EngineException("Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']"
1132                                   .format(vnfpkg_id, kdu_name))
1133         if repo_name:
1134             del filter_q["_id"]
1135             filter_q["name"] = repo_name
1136             repo = self.db.get_one("k8srepos", filter_q)
1137             k8srepo_id = repo.get("_id")
1138             k8srepo_url = repo.get("url")
1139         else:
1140             k8srepo_id = None
1141             k8srepo_url = None
1142         indata["k8srepoId"] = k8srepo_id
1143         indata["k8srepo_url"] = k8srepo_url
1144         vnfpkgop_id = str(uuid4())
1145         vnfpkgop_desc = {
1146             "_id": vnfpkgop_id,
1147             "operationState": "PROCESSING",
1148             "vnfPkgId": vnfpkg_id,
1149             "lcmOperationType": operation,
1150             "isAutomaticInvocation": False,
1151             "isCancelPending": False,
1152             "operationParams": indata,
1153             "links": {
1154                 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id,
1155                 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id,
1156             }
1157         }
1158         self.format_on_new(vnfpkgop_desc, session["project_id"], make_public=session["public"])
1159         ctime = vnfpkgop_desc["_admin"]["created"]
1160         vnfpkgop_desc["statusEnteredTime"] = ctime
1161         vnfpkgop_desc["startTime"] = ctime
1162         self.db.create(self.topic, vnfpkgop_desc)
1163         rollback.append({"topic": self.topic, "_id": vnfpkgop_id})
1164         self.msg.write(self.topic_msg, operation, vnfpkgop_desc)
1165         return vnfpkgop_id, None