osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 import importlib
  20 import copy
  21 # import logging
  22 from hashlib import md5
  23 from osm_common.dbbase import DbException, deep_update_rfc7396
  24 from http import HTTPStatus
  25 from time import time
  26 from uuid import uuid4
  27 from re import fullmatch
  28 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema, \
  29     validate_input, vnfpkgop_new_schema
  30 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  31 etsi_nfv_vnfd = importlib.import_module("osm_im.etsi-nfv-vnfd")
  32 etsi_nfv_nsd = importlib.import_module("osm_im.etsi-nfv-nsd")
  33 from osm_im.nst import nst as nst_im
  34 from pyangbind.lib.serialise import pybindJSONDecoder
  35 import pyangbind.lib.pybindJSON as pybindJSON
  36 from osm_nbi import utils
  37
  38 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  39
  40
  41 class DescriptorTopic(BaseTopic):
  42
  43     def __init__(self, db, fs, msg, auth):
  44         BaseTopic.__init__(self, db, fs, msg, auth)
  45
  46     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  47         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
  48
  49         def _check_unique_id_name(descriptor, position=""):
  50             for desc_key, desc_item in descriptor.items():
  51                 if isinstance(desc_item, list) and desc_item:
  52                     used_ids = []
  53                     desc_item_id = None
  54                     for index, list_item in enumerate(desc_item):
  55                         if isinstance(list_item, dict):
  56                             _check_unique_id_name(list_item, "{}.{}[{}]"
  57                                                   .format(position, desc_key, index))
  58                             # Base case
  59                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  60                                 desc_item_id = "id" if list_item.get("id") else "name"
  61                             if desc_item_id and list_item.get(desc_item_id):
  62                                 if list_item[desc_item_id] in used_ids:
  63                                     position = "{}.{}[{}]".format(position, desc_key, index)
  64                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  65                                                           .format(desc_item_id, list_item[desc_item_id],
  66                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  67                                 used_ids.append(list_item[desc_item_id])
  68
  69         _check_unique_id_name(final_content)
  70         # 1. validate again with pyangbind
  71         # 1.1. remove internal keys
  72         internal_keys = {}
  73         for k in ("_id", "_admin"):
  74             if k in final_content:
  75                 internal_keys[k] = final_content.pop(k)
  76         storage_params = internal_keys["_admin"].get("storage")
  77         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  78
  79         # 1.2. modify final_content with a serialized version
  80         final_content = copy.deepcopy(serialized)
  81         # 1.3. restore internal keys
  82         for k, v in internal_keys.items():
  83             final_content[k] = v
  84         if session["force"]:
  85             return final_content
  86
  87         # 2. check that this id is not present
  88         if "id" in edit_content:
  89             _filter = self._get_project_filter(session)
  90
  91             _filter["id"] = final_content["id"]
  92             _filter["_id.neq"] = _id
  93
  94             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  95                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  96                                                                                                final_content["id"]),
  97                                       HTTPStatus.CONFLICT)
  98
  99         return final_content
 100
 101     @staticmethod
 102     def format_on_new(content, project_id=None, make_public=False):
 103         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 104         content["_admin"]["onboardingState"] = "CREATED"
 105         content["_admin"]["operationalState"] = "DISABLED"
 106         content["_admin"]["usageState"] = "NOT_IN_USE"
 107
 108     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 109         """
 110         Deletes file system storage associated with the descriptor
 111         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 112         :param _id: server internal id
 113         :param db_content: The database content of the descriptor
 114         :param not_send_msg: To not send message (False) or store content (list) instead
 115         :return: None if ok or raises EngineException with the problem
 116         """
 117         self.fs.file_delete(_id, ignore_non_exist=True)
 118         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 119
 120     @staticmethod
 121     def get_one_by_id(db, session, topic, id):
 122         # find owned by this project
 123         _filter = BaseTopic._get_project_filter(session)
 124         _filter["id"] = id
 125         desc_list = db.get_list(topic, _filter)
 126         if len(desc_list) == 1:
 127             return desc_list[0]
 128         elif len(desc_list) > 1:
 129             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 130                               HTTPStatus.CONFLICT)
 131
 132         # not found any: try to find public
 133         _filter = BaseTopic._get_project_filter(session)
 134         _filter["id"] = id
 135         desc_list = db.get_list(topic, _filter)
 136         if not desc_list:
 137             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 138         elif len(desc_list) == 1:
 139             return desc_list[0]
 140         else:
 141             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 142                 topic[:-1], id), HTTPStatus.CONFLICT)
 143
 144     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 145         """
 146         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 147         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 148         (self.upload_content)
 149         :param rollback: list to append created items at database in case a rollback may to be done
 150         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 151         :param indata: data to be inserted
 152         :param kwargs: used to override the indata descriptor
 153         :param headers: http request headers
 154         :return: _id, None: identity of the inserted data; and None as there is not any operation
 155         """
 156
 157         # No needed to capture exceptions
 158         # Check Quota
 159         self.check_quota(session)
 160
 161         # _remove_envelop
 162         if indata:
 163             if "userDefinedData" in indata:
 164                 indata = indata['userDefinedData']
 165
 166         # Override descriptor with query string kwargs
 167         self._update_input_with_kwargs(indata, kwargs)
 168         # uncomment when this method is implemented.
 169         # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 170         # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 171
 172         content = {"_admin": {"userDefinedData": indata}}
 173         self.format_on_new(content, session["project_id"], make_public=session["public"])
 174         _id = self.db.create(self.topic, content)
 175         rollback.append({"topic": self.topic, "_id": _id})
 176         self._send_msg("created", {"_id": _id})
 177         return _id, None
 178
 179     def upload_content(self, session, _id, indata, kwargs, headers):
 180         """
 181         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 182         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 183         :param _id : the nsd,vnfd is already created, this is the id
 184         :param indata: http body request
 185         :param kwargs: user query string to override parameters. NOT USED
 186         :param headers:  http request headers
 187         :return: True if package is completely uploaded or False if partial content has been uploded
 188             Raise exception on error
 189         """
 190         # Check that _id exists and it is valid
 191         current_desc = self.show(session, _id)
 192
 193         content_range_text = headers.get("Content-Range")
 194         expected_md5 = headers.get("Content-File-MD5")
 195         compressed = None
 196         content_type = headers.get("Content-Type")
 197         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 198                 "application/zip" in content_type:
 199             compressed = "gzip"
 200         filename = headers.get("Content-Filename")
 201         if not filename:
 202             filename = "package.tar.gz" if compressed else "package"
 203         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 204         file_pkg = None
 205         error_text = ""
 206         try:
 207             if content_range_text:
 208                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 209                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 210                     raise IndexError()
 211                 start = int(content_range[1])
 212                 end = int(content_range[2]) + 1
 213                 total = int(content_range[3])
 214             else:
 215                 start = 0
 216             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 217
 218             if start:
 219                 if not self.fs.file_exists(temp_folder, 'dir'):
 220                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 221             else:
 222                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 223                 self.fs.mkdir(temp_folder)
 224
 225             storage = self.fs.get_params()
 226             storage["folder"] = _id
 227
 228             file_path = (temp_folder, filename)
 229             if self.fs.file_exists(file_path, 'file'):
 230                 file_size = self.fs.file_size(file_path)
 231             else:
 232                 file_size = 0
 233             if file_size != start:
 234                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 235                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 236             file_pkg = self.fs.file_open(file_path, 'a+b')
 237             if isinstance(indata, dict):
 238                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 239                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 240             else:
 241                 indata_len = 0
 242                 while True:
 243                     indata_text = indata.read(4096)
 244                     indata_len += len(indata_text)
 245                     if not indata_text:
 246                         break
 247                     file_pkg.write(indata_text)
 248             if content_range_text:
 249                 if indata_len != end - start:
 250                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 251                         start, end - 1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 252                 if end != total:
 253                     # TODO update to UPLOADING
 254                     return False
 255
 256             # PACKAGE UPLOADED
 257             if expected_md5:
 258                 file_pkg.seek(0, 0)
 259                 file_md5 = md5()
 260                 chunk_data = file_pkg.read(1024)
 261                 while chunk_data:
 262                     file_md5.update(chunk_data)
 263                     chunk_data = file_pkg.read(1024)
 264                 if expected_md5 != file_md5.hexdigest():
 265                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 266             file_pkg.seek(0, 0)
 267             if compressed == "gzip":
 268                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 269                 descriptor_file_name = None
 270                 for tarinfo in tar:
 271                     tarname = tarinfo.name
 272                     tarname_path = tarname.split("/")
 273                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 274                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 275                     if len(tarname_path) == 1 and not tarinfo.isdir():
 276                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 277                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 278                         storage["pkg-dir"] = tarname_path[0]
 279                         if len(tarname_path) == 2:
 280                             if descriptor_file_name:
 281                                 raise EngineException(
 282                                     "Found more than one descriptor file at package descriptor tar.gz")
 283                             descriptor_file_name = tarname
 284                 if not descriptor_file_name:
 285                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 286                 storage["descriptor"] = descriptor_file_name
 287                 storage["zipfile"] = filename
 288                 self.fs.file_extract(tar, temp_folder)
 289                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 290                     content = descriptor_file.read()
 291             else:
 292                 content = file_pkg.read()
 293                 storage["descriptor"] = descriptor_file_name = filename
 294
 295             if descriptor_file_name.endswith(".json"):
 296                 error_text = "Invalid json format "
 297                 indata = json.load(content)
 298             else:
 299                 error_text = "Invalid yaml format "
 300                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 301
 302             current_desc["_admin"]["storage"] = storage
 303             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 304             current_desc["_admin"]["operationalState"] = "ENABLED"
 305
 306             indata = self._remove_envelop(indata)
 307
 308             # Override descriptor with query string kwargs
 309             if kwargs:
 310                 self._update_input_with_kwargs(indata, kwargs)
 311
 312             deep_update_rfc7396(current_desc, indata)
 313             current_desc = self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 314             current_desc["_admin"]["modified"] = time()
 315             self.db.replace(self.topic, _id, current_desc)
 316             self.fs.dir_rename(temp_folder, _id)
 317
 318             indata["_id"] = _id
 319             self._send_msg("edited", indata)
 320
 321             # TODO if descriptor has changed because kwargs update content and remove cached zip
 322             # TODO if zip is not present creates one
 323             return True
 324
 325         except EngineException:
 326             raise
 327         except IndexError:
 328             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 329                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 330         except IOError as e:
 331             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 332         except tarfile.ReadError as e:
 333             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 334         except (ValueError, yaml.YAMLError) as e:
 335             raise EngineException(error_text + str(e))
 336         except ValidationError as e:
 337             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 338         finally:
 339             if file_pkg:
 340                 file_pkg.close()
 341
 342     def get_file(self, session, _id, path=None, accept_header=None):
 343         """
 344         Return the file content of a vnfd or nsd
 345         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 346         :param _id: Identity of the vnfd, nsd
 347         :param path: artifact path or "$DESCRIPTOR" or None
 348         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 349         :return: opened file plus Accept format or raises an exception
 350         """
 351         accept_text = accept_zip = False
 352         if accept_header:
 353             if 'text/plain' in accept_header or '*/*' in accept_header:
 354                 accept_text = True
 355             if 'application/zip' in accept_header or '*/*' in accept_header:
 356                 accept_zip = 'application/zip'
 357             elif 'application/gzip' in accept_header:
 358                 accept_zip = 'application/gzip'
 359
 360         if not accept_text and not accept_zip:
 361             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 362                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 363
 364         content = self.show(session, _id)
 365         if content["_admin"]["onboardingState"] != "ONBOARDED":
 366             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 367                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 368                                   http_code=HTTPStatus.CONFLICT)
 369         storage = content["_admin"]["storage"]
 370         if path is not None and path != "$DESCRIPTOR":  # artifacts
 371             if not storage.get('pkg-dir'):
 372                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 373             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 374                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 375                 return folder_content, "text/plain"
 376                 # TODO manage folders in http
 377             else:
 378                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \
 379                     "application/octet-stream"
 380
 381         # pkgtype   accept  ZIP  TEXT    -> result
 382         # manyfiles         yes  X       -> zip
 383         #                   no   yes     -> error
 384         # onefile           yes  no      -> zip
 385         #                   X    yes     -> text
 386         contain_many_files = False
 387         if storage.get('pkg-dir'):
 388             # check if there are more than one file in the package, ignoring checksums.txt.
 389             pkg_files = self.fs.dir_ls((storage['folder'], storage['pkg-dir']))
 390             if len(pkg_files) >= 3 or (len(pkg_files) == 2 and 'checksums.txt' not in pkg_files):
 391                 contain_many_files = True
 392         if accept_text and (not contain_many_files or path == "$DESCRIPTOR"):
 393             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 394         elif contain_many_files and not accept_zip:
 395             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 396                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 397         else:
 398             if not storage.get('zipfile'):
 399                 # TODO generate zipfile if not present
 400                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 401                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 402             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 403
 404     def _remove_yang_prefixes_from_descriptor(self, descriptor):
 405         new_descriptor = {}
 406         for k, v in descriptor.items():
 407             new_v = v
 408             if isinstance(v, dict):
 409                 new_v = self._remove_yang_prefixes_from_descriptor(v)
 410             elif isinstance(v, list):
 411                 new_v = list()
 412                 for x in v:
 413                     if isinstance(x, dict):
 414                         new_v.append(self._remove_yang_prefixes_from_descriptor(x))
 415                     else:
 416                         new_v.append(x)
 417             new_descriptor[k.split(':')[-1]] = new_v
 418         return new_descriptor
 419
 420     def pyangbind_validation(self, item, data, force=False):
 421         raise EngineException("Not possible to validate '{}' item".format(item),
 422                               http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 423
 424     def _validate_input_edit(self, indata, content, force=False):
 425         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 426         if "_id" in indata:
 427             indata.pop("_id")
 428         if "_admin" not in indata:
 429             indata["_admin"] = {}
 430
 431         if "operationalState" in indata:
 432             if indata["operationalState"] in ("ENABLED", "DISABLED"):
 433                 indata["_admin"]["operationalState"] = indata.pop("operationalState")
 434             else:
 435                 raise EngineException("State '{}' is not a valid operational state"
 436                                       .format(indata["operationalState"]),
 437                                       http_code=HTTPStatus.BAD_REQUEST)
 438
 439         # In the case of user defined data, we need to put the data in the root of the object
 440         # to preserve current expected behaviour
 441         if "userDefinedData" in indata:
 442             data = indata.pop("userDefinedData")
 443             if type(data) == dict:
 444                 indata["_admin"]["userDefinedData"] = data
 445             else:
 446                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 447                                       .format(type(data)),
 448                                       http_code=HTTPStatus.BAD_REQUEST)
 449
 450         if ("operationalState" in indata["_admin"] and
 451                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 452             raise EngineException("operationalState already {}".format(content["_admin"]["operationalState"]),
 453                                   http_code=HTTPStatus.CONFLICT)
 454
 455         return indata
 456
 457
 458 class VnfdTopic(DescriptorTopic):
 459     topic = "vnfds"
 460     topic_msg = "vnfd"
 461
 462     def __init__(self, db, fs, msg, auth):
 463         DescriptorTopic.__init__(self, db, fs, msg, auth)
 464
 465     def pyangbind_validation(self, item, data, force=False):
 466         if self._descriptor_data_is_in_old_format(data):
 467             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 468                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 469         try:
 470             myvnfd = etsi_nfv_vnfd.etsi_nfv_vnfd()
 471             pybindJSONDecoder.load_ietf_json({'etsi-nfv-vnfd:vnfd': data}, None, None, obj=myvnfd,
 472                                              path_helper=True, skip_unknown=force)
 473             out = pybindJSON.dumps(myvnfd, mode="ietf")
 474             desc_out = self._remove_envelop(yaml.safe_load(out))
 475             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 476             return utils.deep_update_dict(data, desc_out)
 477         except Exception as e:
 478             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 479                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 480
 481     @staticmethod
 482     def _descriptor_data_is_in_old_format(data):
 483         return ('vnfd-catalog' in data) or ('vnfd:vnfd-catalog' in data)
 484
 485     @staticmethod
 486     def _remove_envelop(indata=None):
 487         if not indata:
 488             return {}
 489         clean_indata = indata
 490
 491         if clean_indata.get('etsi-nfv-vnfd:vnfd'):
 492             if not isinstance(clean_indata['etsi-nfv-vnfd:vnfd'], dict):
 493                 raise EngineException("'etsi-nfv-vnfd:vnfd' must be a dict")
 494             clean_indata = clean_indata['etsi-nfv-vnfd:vnfd']
 495         elif clean_indata.get('vnfd'):
 496             if not isinstance(clean_indata['vnfd'], dict):
 497                 raise EngineException("'vnfd' must be dict")
 498             clean_indata = clean_indata['vnfd']
 499
 500         return clean_indata
 501
 502     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 503         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
 504
 505         # set type of vnfd
 506         contains_pdu = False
 507         contains_vdu = False
 508         for vdu in get_iterable(final_content.get("vdu")):
 509             if vdu.get("pdu-type"):
 510                 contains_pdu = True
 511             else:
 512                 contains_vdu = True
 513         if contains_pdu:
 514             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 515         elif contains_vdu:
 516             final_content["_admin"]["type"] = "vnfd"
 517         # if neither vud nor pdu do not fill type
 518         return final_content
 519
 520     def check_conflict_on_del(self, session, _id, db_content):
 521         """
 522         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 523         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 524         that uses this vnfd
 525         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 526         :param _id: vnfd internal id
 527         :param db_content: The database content of the _id.
 528         :return: None or raises EngineException with the conflict
 529         """
 530         if session["force"]:
 531             return
 532         descriptor = db_content
 533         descriptor_id = descriptor.get("id")
 534         if not descriptor_id:  # empty vnfd not uploaded
 535             return
 536
 537         _filter = self._get_project_filter(session)
 538
 539         # check vnfrs using this vnfd
 540         _filter["vnfd-id"] = _id
 541         if self.db.get_list("vnfrs", _filter):
 542             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 543
 544         # check NSD referencing this VNFD
 545         del _filter["vnfd-id"]
 546         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 547         if self.db.get_list("nsds", _filter):
 548             raise EngineException("There is at least one NSD referencing this descriptor",
 549                                   http_code=HTTPStatus.CONFLICT)
 550
 551     def _validate_input_new(self, indata, storage_params, force=False):
 552         indata.pop("onboardingState", None)
 553         indata.pop("operationalState", None)
 554         indata.pop("usageState", None)
 555         indata.pop("links", None)
 556
 557         indata = self.pyangbind_validation("vnfds", indata, force)
 558         # Cross references validation in the descriptor
 559
 560         self.validate_mgmt_interface_connection_point(indata)
 561
 562         for vdu in get_iterable(indata.get("vdu")):
 563             self.validate_vdu_internal_connection_points(vdu)
 564             self._validate_vdu_cloud_init_in_package(storage_params, vdu, indata)
 565         self._validate_vdu_charms_in_package(storage_params, indata)
 566
 567         self._validate_vnf_charms_in_package(storage_params, indata)
 568
 569         self.validate_external_connection_points(indata)
 570         self.validate_internal_virtual_links(indata)
 571         self.validate_monitoring_params(indata)
 572         self.validate_scaling_group_descriptor(indata)
 573
 574         return indata
 575
 576     @staticmethod
 577     def validate_mgmt_interface_connection_point(indata):
 578         if not indata.get("vdu"):
 579             return
 580         if not indata.get("mgmt-cp"):
 581             raise EngineException("'mgmt-cp' is a mandatory field and it is not defined",
 582                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 583
 584         for cp in get_iterable(indata.get("ext-cpd")):
 585             if cp["id"] == indata["mgmt-cp"]:
 586                 break
 587         else:
 588             raise EngineException("mgmt-cp='{}' must match an existing ext-cpd".format(indata["mgmt-cp"]),
 589                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 590
 591     @staticmethod
 592     def validate_vdu_internal_connection_points(vdu):
 593         int_cpds = set()
 594         for cpd in get_iterable(vdu.get("int-cpd")):
 595             cpd_id = cpd.get("id")
 596             if cpd_id and cpd_id in int_cpds:
 597                 raise EngineException("vdu[id='{}']:int-cpd[id='{}'] is already used by other int-cpd"
 598                                       .format(vdu["id"], cpd_id),
 599                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 600             int_cpds.add(cpd_id)
 601
 602     @staticmethod
 603     def validate_external_connection_points(indata):
 604         all_vdus_int_cpds = set()
 605         for vdu in get_iterable(indata.get("vdu")):
 606             for int_cpd in get_iterable(vdu.get("int-cpd")):
 607                 all_vdus_int_cpds.add((vdu.get("id"), int_cpd.get("id")))
 608
 609         ext_cpds = set()
 610         for cpd in get_iterable(indata.get("ext-cpd")):
 611             cpd_id = cpd.get("id")
 612             if cpd_id and cpd_id in ext_cpds:
 613                 raise EngineException("ext-cpd[id='{}'] is already used by other ext-cpd".format(cpd_id),
 614                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 615             ext_cpds.add(cpd_id)
 616
 617             int_cpd = cpd.get("int-cpd")
 618             if int_cpd:
 619                 if (int_cpd.get("vdu-id"), int_cpd.get("cpd")) not in all_vdus_int_cpds:
 620                     raise EngineException("ext-cpd[id='{}']:int-cpd must match an existing vdu int-cpd".format(cpd_id),
 621                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 622             # TODO: Validate k8s-cluster-net points to a valid k8s-cluster:nets ?
 623
 624     def _validate_vdu_charms_in_package(self, storage_params, indata):
 625         for df in indata["df"]:
 626             if "lcm-operations-configuration" in df and "operate-vnf-op-config" in df["lcm-operations-configuration"]:
 627                 configs = df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", [])
 628                 for config in configs:
 629                     if config.get("juju"):
 630                         if not self._validate_package_folders(storage_params, 'charms'):
 631                             raise EngineException("Charm defined in vnf[id={}] but not present in "
 632                                                   "package".format(indata["id"]))
 633
 634     def _validate_vdu_cloud_init_in_package(self, storage_params, vdu, indata):
 635         if not vdu.get("cloud-init-file"):
 636             return
 637         if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 638             raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 639                                   "package".format(indata["id"], vdu["id"]))
 640
 641     def _validate_vnf_charms_in_package(self, storage_params, indata):
 642         # Get VNF configuration through new container
 643         for deployment_flavor in indata.get('df', []):
 644             if "lcm-operations-configuration" not in deployment_flavor:
 645                 return
 646             if "operate-vnf-op-config" not in deployment_flavor["lcm-operations-configuration"]:
 647                 return
 648             for day_1_2_config in deployment_flavor["lcm-operations-configuration"]["operate-vnf-op-config"]["day1-2"]:
 649                 if day_1_2_config["id"] == indata["id"]:
 650                     vnf_configuration = day_1_2_config
 651                     if vnf_configuration.get("juju"):
 652                         if not self._validate_package_folders(storage_params, 'charms'):
 653                             raise EngineException("Charm defined in vnf[id={}] but not present in "
 654                                                   "package".format(indata["id"]))
 655
 656     def _validate_package_folders(self, storage_params, folder, file=None):
 657         if not storage_params or not storage_params.get("pkg-dir"):
 658             return False
 659         else:
 660             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 661                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 662             else:
 663                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 664             if file:
 665                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 666             else:
 667                 if self.fs.file_exists(f, 'dir'):
 668                     if self.fs.dir_ls(f):
 669                         return True
 670             return False
 671
 672     @staticmethod
 673     def validate_internal_virtual_links(indata):
 674         all_ivld_ids = set()
 675         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 676             ivld_id = ivld.get("id")
 677             if ivld_id and ivld_id in all_ivld_ids:
 678                 raise EngineException("Duplicated VLD id in int-virtual-link-desc[id={}]".format(ivld_id),
 679                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 680             else:
 681                 all_ivld_ids.add(ivld_id)
 682
 683         for vdu in get_iterable(indata.get("vdu")):
 684             for int_cpd in get_iterable(vdu.get("int-cpd")):
 685                 int_cpd_ivld_id = int_cpd.get("int-virtual-link-desc")
 686                 if int_cpd_ivld_id and int_cpd_ivld_id not in all_ivld_ids:
 687                     raise EngineException(
 688                         "vdu[id='{}']:int-cpd[id='{}']:int-virtual-link-desc='{}' must match an existing "
 689                         "int-virtual-link-desc".format(vdu["id"], int_cpd["id"], int_cpd_ivld_id),
 690                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 691
 692         for df in get_iterable(indata.get("df")):
 693             for vlp in get_iterable(df.get("virtual-link-profile")):
 694                 vlp_ivld_id = vlp.get("id")
 695                 if vlp_ivld_id and vlp_ivld_id not in all_ivld_ids:
 696                     raise EngineException("df[id='{}']:virtual-link-profile='{}' must match an existing "
 697                                           "int-virtual-link-desc".format(df["id"], vlp_ivld_id),
 698                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 699
 700     @staticmethod
 701     def validate_monitoring_params(indata):
 702         all_monitoring_params = set()
 703         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 704             for mp in get_iterable(ivld.get("monitoring-parameters")):
 705                 mp_id = mp.get("id")
 706                 if mp_id and mp_id in all_monitoring_params:
 707                     raise EngineException("Duplicated monitoring-parameter id in "
 708                                           "int-virtual-link-desc[id='{}']:monitoring-parameters[id='{}']"
 709                                           .format(ivld["id"], mp_id),
 710                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 711                 else:
 712                     all_monitoring_params.add(mp_id)
 713
 714         for vdu in get_iterable(indata.get("vdu")):
 715             for mp in get_iterable(vdu.get("monitoring-parameter")):
 716                 mp_id = mp.get("id")
 717                 if mp_id and mp_id in all_monitoring_params:
 718                     raise EngineException("Duplicated monitoring-parameter id in "
 719                                           "vdu[id='{}']:monitoring-parameter[id='{}']"
 720                                           .format(vdu["id"], mp_id),
 721                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 722                 else:
 723                     all_monitoring_params.add(mp_id)
 724
 725         for df in get_iterable(indata.get("df")):
 726             for mp in get_iterable(df.get("monitoring-parameter")):
 727                 mp_id = mp.get("id")
 728                 if mp_id and mp_id in all_monitoring_params:
 729                     raise EngineException("Duplicated monitoring-parameter id in "
 730                                           "df[id='{}']:monitoring-parameter[id='{}']"
 731                                           .format(df["id"], mp_id),
 732                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 733                 else:
 734                     all_monitoring_params.add(mp_id)
 735
 736     @staticmethod
 737     def validate_scaling_group_descriptor(indata):
 738         all_monitoring_params = set()
 739         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 740             for mp in get_iterable(ivld.get("monitoring-parameters")):
 741                 all_monitoring_params.add(mp.get("id"))
 742
 743         for vdu in get_iterable(indata.get("vdu")):
 744             for mp in get_iterable(vdu.get("monitoring-parameter")):
 745                 all_monitoring_params.add(mp.get("id"))
 746
 747         for df in get_iterable(indata.get("df")):
 748             for mp in get_iterable(df.get("monitoring-parameter")):
 749                 all_monitoring_params.add(mp.get("id"))
 750
 751         for df in get_iterable(indata.get("df")):
 752             for sa in get_iterable(df.get("scaling-aspect")):
 753                 for sp in get_iterable(sa.get("scaling-policy")):
 754                     for sc in get_iterable(sp.get("scaling-criteria")):
 755                         sc_monitoring_param = sc.get("vnf-monitoring-param-ref")
 756                         if sc_monitoring_param and sc_monitoring_param not in all_monitoring_params:
 757                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-policy"
 758                                                   "[name='{}']:scaling-criteria[name='{}']: "
 759                                                   "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 760                                                   .format(df["id"], sa["id"], sp["name"], sc["name"],
 761                                                           sc_monitoring_param),
 762                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 763
 764                 for sca in get_iterable(sa.get("scaling-config-action")):
 765                     if "lcm-operations-configuration" not in df \
 766                         or "operate-vnf-op-config" not in df["lcm-operations-configuration"] \
 767                         or not utils.find_in_list(
 768                             df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", []),
 769                             lambda config: config["id"] == indata["id"]):
 770                         raise EngineException("'day1-2 configuration' not defined in the descriptor but it is "
 771                                               "referenced by df[id='{}']:scaling-aspect[id='{}']:scaling-config-action"
 772                                               .format(df["id"], sa["id"]),
 773                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 774                     for configuration in get_iterable(
 775                         df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", [])
 776                     ):
 777                         for primitive in get_iterable(configuration.get("config-primitive")):
 778                             if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 779                                 break
 780                         else:
 781                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-config-action:vnf-"
 782                                                   "config-primitive-name-ref='{}' does not match any "
 783                                                   "day1-2 configuration:config-primitive:name"
 784                                                   .format(df["id"], sa["id"], sca["vnf-config-primitive-name-ref"]),
 785                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 786
 787     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 788         """
 789         Deletes associate file system storage (via super)
 790         Deletes associated vnfpkgops from database.
 791         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 792         :param _id: server internal id
 793         :param db_content: The database content of the descriptor
 794         :return: None
 795         :raises: FsException in case of error while deleting associated storage
 796         """
 797         super().delete_extra(session, _id, db_content, not_send_msg)
 798         self.db.del_list("vnfpkgops", {"vnfPkgId": _id})
 799
 800     def sol005_projection(self, data):
 801         data["onboardingState"] = data["_admin"]["onboardingState"]
 802         data["operationalState"] = data["_admin"]["operationalState"]
 803         data["usageState"] = data["_admin"]["usageState"]
 804
 805         links = {}
 806         links["self"] = {"href": "/vnfpkgm/v1/vnf_packages/{}".format(data["_id"])}
 807         links["vnfd"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(data["_id"])}
 808         links["packageContent"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/package_content".format(data["_id"])}
 809         data["_links"] = links
 810
 811         return super().sol005_projection(data)
 812
 813
 814 class NsdTopic(DescriptorTopic):
 815     topic = "nsds"
 816     topic_msg = "nsd"
 817
 818     def __init__(self, db, fs, msg, auth):
 819         DescriptorTopic.__init__(self, db, fs, msg, auth)
 820
 821     def pyangbind_validation(self, item, data, force=False):
 822         if self._descriptor_data_is_in_old_format(data):
 823             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 824                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 825         try:
 826             nsd_vnf_profiles = data.get('df', [{}])[0].get('vnf-profile', [])
 827             mynsd = etsi_nfv_nsd.etsi_nfv_nsd()
 828             pybindJSONDecoder.load_ietf_json({'nsd': {'nsd': [data]}}, None, None, obj=mynsd,
 829                                              path_helper=True, skip_unknown=force)
 830             out = pybindJSON.dumps(mynsd, mode="ietf")
 831             desc_out = self._remove_envelop(yaml.safe_load(out))
 832             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 833             if nsd_vnf_profiles:
 834                 desc_out['df'][0]['vnf-profile'] = nsd_vnf_profiles
 835             return desc_out
 836         except Exception as e:
 837             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 838                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 839
 840     @staticmethod
 841     def _descriptor_data_is_in_old_format(data):
 842         return ('nsd-catalog' in data) or ('nsd:nsd-catalog' in data)
 843
 844     @staticmethod
 845     def _remove_envelop(indata=None):
 846         if not indata:
 847             return {}
 848         clean_indata = indata
 849
 850         if clean_indata.get('nsd'):
 851             clean_indata = clean_indata['nsd']
 852         elif clean_indata.get('etsi-nfv-nsd:nsd'):
 853             clean_indata = clean_indata['etsi-nfv-nsd:nsd']
 854         if clean_indata.get('nsd'):
 855             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 856                 raise EngineException("'nsd' must be a list of only one element")
 857             clean_indata = clean_indata['nsd'][0]
 858         return clean_indata
 859
 860     def _validate_input_new(self, indata, storage_params, force=False):
 861         indata.pop("nsdOnboardingState", None)
 862         indata.pop("nsdOperationalState", None)
 863         indata.pop("nsdUsageState", None)
 864
 865         indata.pop("links", None)
 866
 867         indata = self.pyangbind_validation("nsds", indata, force)
 868         # Cross references validation in the descriptor
 869         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 870         for vld in get_iterable(indata.get("virtual-link-desc")):
 871             self.validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata)
 872
 873         self.validate_vnf_profiles_vnfd_id(indata)
 874
 875         return indata
 876
 877     @staticmethod
 878     def validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata):
 879         if not vld.get("mgmt-network"):
 880             return
 881         vld_id = vld.get("id")
 882         for df in get_iterable(indata.get("df")):
 883             for vlp in get_iterable(df.get("virtual-link-profile")):
 884                 if vld_id and vld_id == vlp.get("virtual-link-desc-id"):
 885                     if vlp.get("virtual-link-protocol-data"):
 886                         raise EngineException("Error at df[id='{}']:virtual-link-profile[id='{}']:virtual-link-"
 887                                               "protocol-data You cannot set a virtual-link-protocol-data "
 888                                               "when mgmt-network is True"
 889                                               .format(df["id"], vlp["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 890
 891     @staticmethod
 892     def validate_vnf_profiles_vnfd_id(indata):
 893         all_vnfd_ids = set(get_iterable(indata.get("vnfd-id")))
 894         for df in get_iterable(indata.get("df")):
 895             for vnf_profile in get_iterable(df.get("vnf-profile")):
 896                 vnfd_id = vnf_profile.get("vnfd-id")
 897                 if vnfd_id and vnfd_id not in all_vnfd_ids:
 898                     raise EngineException("Error at df[id='{}']:vnf_profile[id='{}']:vnfd-id='{}' "
 899                                           "does not match any vnfd-id".format(df["id"], vnf_profile["id"], vnfd_id),
 900                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 901
 902     def _validate_input_edit(self, indata, content, force=False):
 903         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 904         """
 905         indata looks as follows:
 906             - In the new case (conformant)
 907                 {'nsdOperationalState': 'DISABLED', 'userDefinedData': {'id': 'string23',
 908                 '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}}
 909             - In the old case (backwards-compatible)
 910                 {'id': 'string23', '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}
 911         """
 912         if "_admin" not in indata:
 913             indata["_admin"] = {}
 914
 915         if "nsdOperationalState" in indata:
 916             if indata["nsdOperationalState"] in ("ENABLED", "DISABLED"):
 917                 indata["_admin"]["operationalState"] = indata.pop("nsdOperationalState")
 918             else:
 919                 raise EngineException("State '{}' is not a valid operational state"
 920                                       .format(indata["nsdOperationalState"]),
 921                                       http_code=HTTPStatus.BAD_REQUEST)
 922
 923         # In the case of user defined data, we need to put the data in the root of the object
 924         # to preserve current expected behaviour
 925         if "userDefinedData" in indata:
 926             data = indata.pop("userDefinedData")
 927             if type(data) == dict:
 928                 indata["_admin"]["userDefinedData"] = data
 929             else:
 930                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 931                                       .format(type(data)),
 932                                       http_code=HTTPStatus.BAD_REQUEST)
 933         if ("operationalState" in indata["_admin"] and
 934                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 935             raise EngineException("nsdOperationalState already {}".format(content["_admin"]["operationalState"]),
 936                                   http_code=HTTPStatus.CONFLICT)
 937         return indata
 938
 939     def _check_descriptor_dependencies(self, session, descriptor):
 940         """
 941         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 942         connection points are ok
 943         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 944         :param descriptor: descriptor to be inserted or edit
 945         :return: None or raises exception
 946         """
 947         if session["force"]:
 948             return
 949         vnfds_index = self._get_descriptor_constituent_vnfds_index(session, descriptor)
 950
 951         # Cross references validation in the descriptor and vnfd connection point validation
 952         for df in get_iterable(descriptor.get("df")):
 953             self.validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index)
 954
 955     def _get_descriptor_constituent_vnfds_index(self, session, descriptor):
 956         vnfds_index = {}
 957         if descriptor.get("vnfd-id") and not session["force"]:
 958             for vnfd_id in get_iterable(descriptor.get("vnfd-id")):
 959                 query_filter = self._get_project_filter(session)
 960                 query_filter["id"] = vnfd_id
 961                 vnf_list = self.db.get_list("vnfds", query_filter)
 962                 if not vnf_list:
 963                     raise EngineException("Descriptor error at 'vnfd-id'='{}' references a non "
 964                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 965                 vnfds_index[vnfd_id] = vnf_list[0]
 966         return vnfds_index
 967
 968     @staticmethod
 969     def validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index):
 970         for vnf_profile in get_iterable(df.get("vnf-profile")):
 971             vnfd = vnfds_index.get(vnf_profile["vnfd-id"])
 972             all_vnfd_ext_cpds = set()
 973             for ext_cpd in get_iterable(vnfd.get("ext-cpd")):
 974                 if ext_cpd.get('id'):
 975                     all_vnfd_ext_cpds.add(ext_cpd.get('id'))
 976
 977             for virtual_link in get_iterable(vnf_profile.get("virtual-link-connectivity")):
 978                 for vl_cpd in get_iterable(virtual_link.get("constituent-cpd-id")):
 979                     vl_cpd_id = vl_cpd.get('constituent-cpd-id')
 980                     if vl_cpd_id and vl_cpd_id not in all_vnfd_ext_cpds:
 981                         raise EngineException("Error at df[id='{}']:vnf-profile[id='{}']:virtual-link-connectivity"
 982                                               "[virtual-link-profile-id='{}']:constituent-cpd-id='{}' references a "
 983                                               "non existing ext-cpd:id inside vnfd '{}'"
 984                                               .format(df["id"], vnf_profile["id"],
 985                                                       virtual_link["virtual-link-profile-id"], vl_cpd_id, vnfd["id"]),
 986                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 987
 988     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 989         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
 990
 991         self._check_descriptor_dependencies(session, final_content)
 992
 993         return final_content
 994
 995     def check_conflict_on_del(self, session, _id, db_content):
 996         """
 997         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 998         that NSD can be public and be used by other projects.
 999         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1000         :param _id: nsd internal id
1001         :param db_content: The database content of the _id
1002         :return: None or raises EngineException with the conflict
1003         """
1004         if session["force"]:
1005             return
1006         descriptor = db_content
1007         descriptor_id = descriptor.get("id")
1008         if not descriptor_id:  # empty nsd not uploaded
1009             return
1010
1011         # check NSD used by NS
1012         _filter = self._get_project_filter(session)
1013         _filter["nsd-id"] = _id
1014         if self.db.get_list("nsrs", _filter):
1015             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
1016
1017         # check NSD referenced by NST
1018         del _filter["nsd-id"]
1019         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
1020         if self.db.get_list("nsts", _filter):
1021             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
1022                                   http_code=HTTPStatus.CONFLICT)
1023
1024     def sol005_projection(self, data):
1025         data["nsdOnboardingState"] = data["_admin"]["onboardingState"]
1026         data["nsdOperationalState"] = data["_admin"]["operationalState"]
1027         data["nsdUsageState"] = data["_admin"]["usageState"]
1028
1029         links = {}
1030         links["self"] = {"href": "/nsd/v1/ns_descriptors/{}".format(data["_id"])}
1031         links["nsd_content"] = {"href": "/nsd/v1/ns_descriptors/{}/nsd_content".format(data["_id"])}
1032         data["_links"] = links
1033
1034         return super().sol005_projection(data)
1035
1036
1037 class NstTopic(DescriptorTopic):
1038     topic = "nsts"
1039     topic_msg = "nst"
1040     quota_name = "slice_templates"
1041
1042     def __init__(self, db, fs, msg, auth):
1043         DescriptorTopic.__init__(self, db, fs, msg, auth)
1044
1045     def pyangbind_validation(self, item, data, force=False):
1046         try:
1047             mynst = nst_im()
1048             pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
1049                                              path_helper=True, skip_unknown=force)
1050             out = pybindJSON.dumps(mynst, mode="ietf")
1051             desc_out = self._remove_envelop(yaml.safe_load(out))
1052             return desc_out
1053         except Exception as e:
1054             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
1055                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
1056
1057     @staticmethod
1058     def _remove_envelop(indata=None):
1059         if not indata:
1060             return {}
1061         clean_indata = indata
1062
1063         if clean_indata.get('nst'):
1064             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
1065                 raise EngineException("'nst' must be a list only one element")
1066             clean_indata = clean_indata['nst'][0]
1067         elif clean_indata.get('nst:nst'):
1068             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
1069                 raise EngineException("'nst:nst' must be a list only one element")
1070             clean_indata = clean_indata['nst:nst'][0]
1071         return clean_indata
1072
1073     def _validate_input_new(self, indata, storage_params, force=False):
1074         indata.pop("onboardingState", None)
1075         indata.pop("operationalState", None)
1076         indata.pop("usageState", None)
1077         indata = self.pyangbind_validation("nsts", indata, force)
1078         return indata.copy()
1079
1080     def _check_descriptor_dependencies(self, session, descriptor):
1081         """
1082         Check that the dependent descriptors exist on a new descriptor or edition
1083         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1084         :param descriptor: descriptor to be inserted or edit
1085         :return: None or raises exception
1086         """
1087         if not descriptor.get("netslice-subnet"):
1088             return
1089         for nsd in descriptor["netslice-subnet"]:
1090             nsd_id = nsd["nsd-ref"]
1091             filter_q = self._get_project_filter(session)
1092             filter_q["id"] = nsd_id
1093             if not self.db.get_list("nsds", filter_q):
1094                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
1095                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
1096
1097     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
1098         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
1099
1100         self._check_descriptor_dependencies(session, final_content)
1101         return final_content
1102
1103     def check_conflict_on_del(self, session, _id, db_content):
1104         """
1105         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
1106         that NST can be public and be used by other projects.
1107         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1108         :param _id: nst internal id
1109         :param db_content: The database content of the _id.
1110         :return: None or raises EngineException with the conflict
1111         """
1112         # TODO: Check this method
1113         if session["force"]:
1114             return
1115         # Get Network Slice Template from Database
1116         _filter = self._get_project_filter(session)
1117         _filter["_admin.nst-id"] = _id
1118         if self.db.get_list("nsis", _filter):
1119             raise EngineException("there is at least one Netslice Instance using this descriptor",
1120                                   http_code=HTTPStatus.CONFLICT)
1121
1122     def sol005_projection(self, data):
1123         data["onboardingState"] = data["_admin"]["onboardingState"]
1124         data["operationalState"] = data["_admin"]["operationalState"]
1125         data["usageState"] = data["_admin"]["usageState"]
1126
1127         links = {}
1128         links["self"] = {"href": "/nst/v1/netslice_templates/{}".format(data["_id"])}
1129         links["nst"] = {"href": "/nst/v1/netslice_templates/{}/nst".format(data["_id"])}
1130         data["_links"] = links
1131
1132         return super().sol005_projection(data)
1133
1134
1135 class PduTopic(BaseTopic):
1136     topic = "pdus"
1137     topic_msg = "pdu"
1138     quota_name = "pduds"
1139     schema_new = pdu_new_schema
1140     schema_edit = pdu_edit_schema
1141
1142     def __init__(self, db, fs, msg, auth):
1143         BaseTopic.__init__(self, db, fs, msg, auth)
1144
1145     @staticmethod
1146     def format_on_new(content, project_id=None, make_public=False):
1147         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
1148         content["_admin"]["onboardingState"] = "CREATED"
1149         content["_admin"]["operationalState"] = "ENABLED"
1150         content["_admin"]["usageState"] = "NOT_IN_USE"
1151
1152     def check_conflict_on_del(self, session, _id, db_content):
1153         """
1154         Check that there is not any vnfr that uses this PDU
1155         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1156         :param _id: pdu internal id
1157         :param db_content: The database content of the _id.
1158         :return: None or raises EngineException with the conflict
1159         """
1160         if session["force"]:
1161             return
1162
1163         _filter = self._get_project_filter(session)
1164         _filter["vdur.pdu-id"] = _id
1165         if self.db.get_list("vnfrs", _filter):
1166             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)
1167
1168
1169 class VnfPkgOpTopic(BaseTopic):
1170     topic = "vnfpkgops"
1171     topic_msg = "vnfd"
1172     schema_new = vnfpkgop_new_schema
1173     schema_edit = None
1174
1175     def __init__(self, db, fs, msg, auth):
1176         BaseTopic.__init__(self, db, fs, msg, auth)
1177
1178     def edit(self, session, _id, indata=None, kwargs=None, content=None):
1179         raise EngineException("Method 'edit' not allowed for topic '{}'".format(self.topic),
1180                               HTTPStatus.METHOD_NOT_ALLOWED)
1181
1182     def delete(self, session, _id, dry_run=False):
1183         raise EngineException("Method 'delete' not allowed for topic '{}'".format(self.topic),
1184                               HTTPStatus.METHOD_NOT_ALLOWED)
1185
1186     def delete_list(self, session, filter_q=None):
1187         raise EngineException("Method 'delete_list' not allowed for topic '{}'".format(self.topic),
1188                               HTTPStatus.METHOD_NOT_ALLOWED)
1189
1190     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
1191         """
1192         Creates a new entry into database.
1193         :param rollback: list to append created items at database in case a rollback may to be done
1194         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1195         :param indata: data to be inserted
1196         :param kwargs: used to override the indata descriptor
1197         :param headers: http request headers
1198         :return: _id, op_id:
1199             _id: identity of the inserted data.
1200              op_id: None
1201         """
1202         self._update_input_with_kwargs(indata, kwargs)
1203         validate_input(indata, self.schema_new)
1204         vnfpkg_id = indata["vnfPkgId"]
1205         filter_q = BaseTopic._get_project_filter(session)
1206         filter_q["_id"] = vnfpkg_id
1207         vnfd = self.db.get_one("vnfds", filter_q)
1208         operation = indata["lcmOperationType"]
1209         kdu_name = indata["kdu_name"]
1210         for kdu in vnfd.get("kdu", []):
1211             if kdu["name"] == kdu_name:
1212                 helm_chart = kdu.get("helm-chart")
1213                 juju_bundle = kdu.get("juju-bundle")
1214                 break
1215         else:
1216             raise EngineException("Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id, kdu_name))
1217         if helm_chart:
1218             indata["helm-chart"] = helm_chart
1219             match = fullmatch(r"([^/]*)/([^/]*)", helm_chart)
1220             repo_name = match.group(1) if match else None
1221         elif juju_bundle:
1222             indata["juju-bundle"] = juju_bundle
1223             match = fullmatch(r"([^/]*)/([^/]*)", juju_bundle)
1224             repo_name = match.group(1) if match else None
1225         else:
1226             raise EngineException("Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']"
1227                                   .format(vnfpkg_id, kdu_name))
1228         if repo_name:
1229             del filter_q["_id"]
1230             filter_q["name"] = repo_name
1231             repo = self.db.get_one("k8srepos", filter_q)
1232             k8srepo_id = repo.get("_id")
1233             k8srepo_url = repo.get("url")
1234         else:
1235             k8srepo_id = None
1236             k8srepo_url = None
1237         indata["k8srepoId"] = k8srepo_id
1238         indata["k8srepo_url"] = k8srepo_url
1239         vnfpkgop_id = str(uuid4())
1240         vnfpkgop_desc = {
1241             "_id": vnfpkgop_id,
1242             "operationState": "PROCESSING",
1243             "vnfPkgId": vnfpkg_id,
1244             "lcmOperationType": operation,
1245             "isAutomaticInvocation": False,
1246             "isCancelPending": False,
1247             "operationParams": indata,
1248             "links": {
1249                 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id,
1250                 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id,
1251             }
1252         }
1253         self.format_on_new(vnfpkgop_desc, session["project_id"], make_public=session["public"])
1254         ctime = vnfpkgop_desc["_admin"]["created"]
1255         vnfpkgop_desc["statusEnteredTime"] = ctime
1256         vnfpkgop_desc["startTime"] = ctime
1257         self.db.create(self.topic, vnfpkgop_desc)
1258         rollback.append({"topic": self.topic, "_id": vnfpkgop_id})
1259         self.msg.write(self.topic_msg, operation, vnfpkgop_desc)
1260         return vnfpkgop_id, None