osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 import importlib
  20 import copy
  21 # import logging
  22 from hashlib import md5
  23 from osm_common.dbbase import DbException, deep_update_rfc7396
  24 from http import HTTPStatus
  25 from time import time
  26 from uuid import uuid4
  27 from re import fullmatch
  28 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema, \
  29     validate_input, vnfpkgop_new_schema
  30 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  31 etsi_nfv_vnfd = importlib.import_module("osm_im.etsi-nfv-vnfd")
  32 etsi_nfv_nsd = importlib.import_module("osm_im.etsi-nfv-nsd")
  33 from osm_im.nst import nst as nst_im
  34 from pyangbind.lib.serialise import pybindJSONDecoder
  35 import pyangbind.lib.pybindJSON as pybindJSON
  36 from osm_nbi import utils
  37
  38 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  39
  40
  41 class DescriptorTopic(BaseTopic):
  42
  43     def __init__(self, db, fs, msg, auth):
  44         BaseTopic.__init__(self, db, fs, msg, auth)
  45
  46     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  47         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
  48
  49         def _check_unique_id_name(descriptor, position=""):
  50             for desc_key, desc_item in descriptor.items():
  51                 if isinstance(desc_item, list) and desc_item:
  52                     used_ids = []
  53                     desc_item_id = None
  54                     for index, list_item in enumerate(desc_item):
  55                         if isinstance(list_item, dict):
  56                             _check_unique_id_name(list_item, "{}.{}[{}]"
  57                                                   .format(position, desc_key, index))
  58                             # Base case
  59                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  60                                 desc_item_id = "id" if list_item.get("id") else "name"
  61                             if desc_item_id and list_item.get(desc_item_id):
  62                                 if list_item[desc_item_id] in used_ids:
  63                                     position = "{}.{}[{}]".format(position, desc_key, index)
  64                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  65                                                           .format(desc_item_id, list_item[desc_item_id],
  66                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  67                                 used_ids.append(list_item[desc_item_id])
  68
  69         _check_unique_id_name(final_content)
  70         # 1. validate again with pyangbind
  71         # 1.1. remove internal keys
  72         internal_keys = {}
  73         for k in ("_id", "_admin"):
  74             if k in final_content:
  75                 internal_keys[k] = final_content.pop(k)
  76         storage_params = internal_keys["_admin"].get("storage")
  77         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  78
  79         # 1.2. modify final_content with a serialized version
  80         final_content = copy.deepcopy(serialized)
  81         # 1.3. restore internal keys
  82         for k, v in internal_keys.items():
  83             final_content[k] = v
  84         if session["force"]:
  85             return final_content
  86
  87         # 2. check that this id is not present
  88         if "id" in edit_content:
  89             _filter = self._get_project_filter(session)
  90
  91             _filter["id"] = final_content["id"]
  92             _filter["_id.neq"] = _id
  93
  94             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  95                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  96                                                                                                final_content["id"]),
  97                                       HTTPStatus.CONFLICT)
  98
  99         return final_content
 100
 101     @staticmethod
 102     def format_on_new(content, project_id=None, make_public=False):
 103         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 104         content["_admin"]["onboardingState"] = "CREATED"
 105         content["_admin"]["operationalState"] = "DISABLED"
 106         content["_admin"]["usageState"] = "NOT_IN_USE"
 107
 108     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 109         """
 110         Deletes file system storage associated with the descriptor
 111         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 112         :param _id: server internal id
 113         :param db_content: The database content of the descriptor
 114         :param not_send_msg: To not send message (False) or store content (list) instead
 115         :return: None if ok or raises EngineException with the problem
 116         """
 117         self.fs.file_delete(_id, ignore_non_exist=True)
 118         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 119
 120     @staticmethod
 121     def get_one_by_id(db, session, topic, id):
 122         # find owned by this project
 123         _filter = BaseTopic._get_project_filter(session)
 124         _filter["id"] = id
 125         desc_list = db.get_list(topic, _filter)
 126         if len(desc_list) == 1:
 127             return desc_list[0]
 128         elif len(desc_list) > 1:
 129             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 130                               HTTPStatus.CONFLICT)
 131
 132         # not found any: try to find public
 133         _filter = BaseTopic._get_project_filter(session)
 134         _filter["id"] = id
 135         desc_list = db.get_list(topic, _filter)
 136         if not desc_list:
 137             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 138         elif len(desc_list) == 1:
 139             return desc_list[0]
 140         else:
 141             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 142                 topic[:-1], id), HTTPStatus.CONFLICT)
 143
 144     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 145         """
 146         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 147         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 148         (self.upload_content)
 149         :param rollback: list to append created items at database in case a rollback may to be done
 150         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 151         :param indata: data to be inserted
 152         :param kwargs: used to override the indata descriptor
 153         :param headers: http request headers
 154         :return: _id, None: identity of the inserted data; and None as there is not any operation
 155         """
 156
 157         # No needed to capture exceptions
 158         # Check Quota
 159         self.check_quota(session)
 160
 161         # _remove_envelop
 162         if indata:
 163             if "userDefinedData" in indata:
 164                 indata = indata['userDefinedData']
 165
 166         # Override descriptor with query string kwargs
 167         self._update_input_with_kwargs(indata, kwargs)
 168         # uncomment when this method is implemented.
 169         # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 170         # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 171
 172         content = {"_admin": {"userDefinedData": indata}}
 173         self.format_on_new(content, session["project_id"], make_public=session["public"])
 174         _id = self.db.create(self.topic, content)
 175         rollback.append({"topic": self.topic, "_id": _id})
 176         self._send_msg("created", {"_id": _id})
 177         return _id, None
 178
 179     def upload_content(self, session, _id, indata, kwargs, headers):
 180         """
 181         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 182         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 183         :param _id : the nsd,vnfd is already created, this is the id
 184         :param indata: http body request
 185         :param kwargs: user query string to override parameters. NOT USED
 186         :param headers:  http request headers
 187         :return: True if package is completely uploaded or False if partial content has been uploded
 188             Raise exception on error
 189         """
 190         # Check that _id exists and it is valid
 191         current_desc = self.show(session, _id)
 192
 193         content_range_text = headers.get("Content-Range")
 194         expected_md5 = headers.get("Content-File-MD5")
 195         compressed = None
 196         content_type = headers.get("Content-Type")
 197         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 198                 "application/zip" in content_type:
 199             compressed = "gzip"
 200         filename = headers.get("Content-Filename")
 201         if not filename:
 202             filename = "package.tar.gz" if compressed else "package"
 203         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 204         file_pkg = None
 205         error_text = ""
 206         try:
 207             if content_range_text:
 208                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 209                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 210                     raise IndexError()
 211                 start = int(content_range[1])
 212                 end = int(content_range[2]) + 1
 213                 total = int(content_range[3])
 214             else:
 215                 start = 0
 216             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 217
 218             if start:
 219                 if not self.fs.file_exists(temp_folder, 'dir'):
 220                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 221             else:
 222                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 223                 self.fs.mkdir(temp_folder)
 224
 225             storage = self.fs.get_params()
 226             storage["folder"] = _id
 227
 228             file_path = (temp_folder, filename)
 229             if self.fs.file_exists(file_path, 'file'):
 230                 file_size = self.fs.file_size(file_path)
 231             else:
 232                 file_size = 0
 233             if file_size != start:
 234                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 235                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 236             file_pkg = self.fs.file_open(file_path, 'a+b')
 237             if isinstance(indata, dict):
 238                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 239                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 240             else:
 241                 indata_len = 0
 242                 while True:
 243                     indata_text = indata.read(4096)
 244                     indata_len += len(indata_text)
 245                     if not indata_text:
 246                         break
 247                     file_pkg.write(indata_text)
 248             if content_range_text:
 249                 if indata_len != end - start:
 250                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 251                         start, end - 1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 252                 if end != total:
 253                     # TODO update to UPLOADING
 254                     return False
 255
 256             # PACKAGE UPLOADED
 257             if expected_md5:
 258                 file_pkg.seek(0, 0)
 259                 file_md5 = md5()
 260                 chunk_data = file_pkg.read(1024)
 261                 while chunk_data:
 262                     file_md5.update(chunk_data)
 263                     chunk_data = file_pkg.read(1024)
 264                 if expected_md5 != file_md5.hexdigest():
 265                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 266             file_pkg.seek(0, 0)
 267             if compressed == "gzip":
 268                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 269                 descriptor_file_name = None
 270                 for tarinfo in tar:
 271                     tarname = tarinfo.name
 272                     tarname_path = tarname.split("/")
 273                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 274                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 275                     if len(tarname_path) == 1 and not tarinfo.isdir():
 276                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 277                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 278                         storage["pkg-dir"] = tarname_path[0]
 279                         if len(tarname_path) == 2:
 280                             if descriptor_file_name:
 281                                 raise EngineException(
 282                                     "Found more than one descriptor file at package descriptor tar.gz")
 283                             descriptor_file_name = tarname
 284                 if not descriptor_file_name:
 285                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 286                 storage["descriptor"] = descriptor_file_name
 287                 storage["zipfile"] = filename
 288                 self.fs.file_extract(tar, temp_folder)
 289                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 290                     content = descriptor_file.read()
 291             else:
 292                 content = file_pkg.read()
 293                 storage["descriptor"] = descriptor_file_name = filename
 294
 295             if descriptor_file_name.endswith(".json"):
 296                 error_text = "Invalid json format "
 297                 indata = json.load(content)
 298             else:
 299                 error_text = "Invalid yaml format "
 300                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 301
 302             current_desc["_admin"]["storage"] = storage
 303             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 304             current_desc["_admin"]["operationalState"] = "ENABLED"
 305
 306             indata = self._remove_envelop(indata)
 307
 308             # Override descriptor with query string kwargs
 309             if kwargs:
 310                 self._update_input_with_kwargs(indata, kwargs)
 311
 312             deep_update_rfc7396(current_desc, indata)
 313             current_desc = self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 314             current_desc["_admin"]["modified"] = time()
 315             self.db.replace(self.topic, _id, current_desc)
 316             self.fs.dir_rename(temp_folder, _id)
 317
 318             indata["_id"] = _id
 319             self._send_msg("edited", indata)
 320
 321             # TODO if descriptor has changed because kwargs update content and remove cached zip
 322             # TODO if zip is not present creates one
 323             return True
 324
 325         except EngineException:
 326             raise
 327         except IndexError:
 328             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 329                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 330         except IOError as e:
 331             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 332         except tarfile.ReadError as e:
 333             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 334         except (ValueError, yaml.YAMLError) as e:
 335             raise EngineException(error_text + str(e))
 336         except ValidationError as e:
 337             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 338         finally:
 339             if file_pkg:
 340                 file_pkg.close()
 341
 342     def get_file(self, session, _id, path=None, accept_header=None):
 343         """
 344         Return the file content of a vnfd or nsd
 345         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 346         :param _id: Identity of the vnfd, nsd
 347         :param path: artifact path or "$DESCRIPTOR" or None
 348         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 349         :return: opened file plus Accept format or raises an exception
 350         """
 351         accept_text = accept_zip = False
 352         if accept_header:
 353             if 'text/plain' in accept_header or '*/*' in accept_header:
 354                 accept_text = True
 355             if 'application/zip' in accept_header or '*/*' in accept_header:
 356                 accept_zip = 'application/zip'
 357             elif 'application/gzip' in accept_header:
 358                 accept_zip = 'application/gzip'
 359
 360         if not accept_text and not accept_zip:
 361             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 362                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 363
 364         content = self.show(session, _id)
 365         if content["_admin"]["onboardingState"] != "ONBOARDED":
 366             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 367                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 368                                   http_code=HTTPStatus.CONFLICT)
 369         storage = content["_admin"]["storage"]
 370         if path is not None and path != "$DESCRIPTOR":  # artifacts
 371             if not storage.get('pkg-dir'):
 372                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 373             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 374                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 375                 return folder_content, "text/plain"
 376                 # TODO manage folders in http
 377             else:
 378                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \
 379                     "application/octet-stream"
 380
 381         # pkgtype   accept  ZIP  TEXT    -> result
 382         # manyfiles         yes  X       -> zip
 383         #                   no   yes     -> error
 384         # onefile           yes  no      -> zip
 385         #                   X    yes     -> text
 386         contain_many_files = False
 387         if storage.get('pkg-dir'):
 388             # check if there are more than one file in the package, ignoring checksums.txt.
 389             pkg_files = self.fs.dir_ls((storage['folder'], storage['pkg-dir']))
 390             if len(pkg_files) >= 3 or (len(pkg_files) == 2 and 'checksums.txt' not in pkg_files):
 391                 contain_many_files = True
 392         if accept_text and (not contain_many_files or path == "$DESCRIPTOR"):
 393             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 394         elif contain_many_files and not accept_zip:
 395             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 396                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 397         else:
 398             if not storage.get('zipfile'):
 399                 # TODO generate zipfile if not present
 400                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 401                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 402             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 403
 404     def _remove_yang_prefixes_from_descriptor(self, descriptor):
 405         new_descriptor = {}
 406         for k, v in descriptor.items():
 407             new_v = v
 408             if isinstance(v, dict):
 409                 new_v = self._remove_yang_prefixes_from_descriptor(v)
 410             elif isinstance(v, list):
 411                 new_v = list()
 412                 for x in v:
 413                     if isinstance(x, dict):
 414                         new_v.append(self._remove_yang_prefixes_from_descriptor(x))
 415                     else:
 416                         new_v.append(x)
 417             new_descriptor[k.split(':')[-1]] = new_v
 418         return new_descriptor
 419
 420     def pyangbind_validation(self, item, data, force=False):
 421         raise EngineException("Not possible to validate '{}' item".format(item),
 422                               http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 423
 424     def _validate_input_edit(self, indata, content, force=False):
 425         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 426         if "_id" in indata:
 427             indata.pop("_id")
 428         if "_admin" not in indata:
 429             indata["_admin"] = {}
 430
 431         if "operationalState" in indata:
 432             if indata["operationalState"] in ("ENABLED", "DISABLED"):
 433                 indata["_admin"]["operationalState"] = indata.pop("operationalState")
 434             else:
 435                 raise EngineException("State '{}' is not a valid operational state"
 436                                       .format(indata["operationalState"]),
 437                                       http_code=HTTPStatus.BAD_REQUEST)
 438
 439         # In the case of user defined data, we need to put the data in the root of the object
 440         # to preserve current expected behaviour
 441         if "userDefinedData" in indata:
 442             data = indata.pop("userDefinedData")
 443             if type(data) == dict:
 444                 indata["_admin"]["userDefinedData"] = data
 445             else:
 446                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 447                                       .format(type(data)),
 448                                       http_code=HTTPStatus.BAD_REQUEST)
 449
 450         if ("operationalState" in indata["_admin"] and
 451                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 452             raise EngineException("operationalState already {}".format(content["_admin"]["operationalState"]),
 453                                   http_code=HTTPStatus.CONFLICT)
 454
 455         return indata
 456
 457
 458 class VnfdTopic(DescriptorTopic):
 459     topic = "vnfds"
 460     topic_msg = "vnfd"
 461
 462     def __init__(self, db, fs, msg, auth):
 463         DescriptorTopic.__init__(self, db, fs, msg, auth)
 464
 465     def pyangbind_validation(self, item, data, force=False):
 466         if self._descriptor_data_is_in_old_format(data):
 467             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 468                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 469         try:
 470             myvnfd = etsi_nfv_vnfd.etsi_nfv_vnfd()
 471             pybindJSONDecoder.load_ietf_json({'etsi-nfv-vnfd:vnfd': data}, None, None, obj=myvnfd,
 472                                              path_helper=True, skip_unknown=force)
 473             out = pybindJSON.dumps(myvnfd, mode="ietf")
 474             desc_out = self._remove_envelop(yaml.safe_load(out))
 475             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 476             return utils.deep_update_dict(data, desc_out)
 477         except Exception as e:
 478             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 479                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 480
 481     @staticmethod
 482     def _descriptor_data_is_in_old_format(data):
 483         return ('vnfd-catalog' in data) or ('vnfd:vnfd-catalog' in data)
 484
 485     @staticmethod
 486     def _remove_envelop(indata=None):
 487         if not indata:
 488             return {}
 489         clean_indata = indata
 490
 491         if clean_indata.get('etsi-nfv-vnfd:vnfd'):
 492             if not isinstance(clean_indata['etsi-nfv-vnfd:vnfd'], dict):
 493                 raise EngineException("'etsi-nfv-vnfd:vnfd' must be a dict")
 494             clean_indata = clean_indata['etsi-nfv-vnfd:vnfd']
 495         elif clean_indata.get('vnfd'):
 496             if not isinstance(clean_indata['vnfd'], dict):
 497                 raise EngineException("'vnfd' must be dict")
 498             clean_indata = clean_indata['vnfd']
 499
 500         return clean_indata
 501
 502     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 503         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
 504
 505         # set type of vnfd
 506         contains_pdu = False
 507         contains_vdu = False
 508         for vdu in get_iterable(final_content.get("vdu")):
 509             if vdu.get("pdu-type"):
 510                 contains_pdu = True
 511             else:
 512                 contains_vdu = True
 513         if contains_pdu:
 514             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 515         elif contains_vdu:
 516             final_content["_admin"]["type"] = "vnfd"
 517         # if neither vud nor pdu do not fill type
 518         return final_content
 519
 520     def check_conflict_on_del(self, session, _id, db_content):
 521         """
 522         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 523         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 524         that uses this vnfd
 525         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 526         :param _id: vnfd internal id
 527         :param db_content: The database content of the _id.
 528         :return: None or raises EngineException with the conflict
 529         """
 530         if session["force"]:
 531             return
 532         descriptor = db_content
 533         descriptor_id = descriptor.get("id")
 534         if not descriptor_id:  # empty vnfd not uploaded
 535             return
 536
 537         _filter = self._get_project_filter(session)
 538
 539         # check vnfrs using this vnfd
 540         _filter["vnfd-id"] = _id
 541         if self.db.get_list("vnfrs", _filter):
 542             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 543
 544         # check NSD referencing this VNFD
 545         del _filter["vnfd-id"]
 546         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 547         if self.db.get_list("nsds", _filter):
 548             raise EngineException("There is at least one NSD referencing this descriptor",
 549                                   http_code=HTTPStatus.CONFLICT)
 550
 551     def _validate_input_new(self, indata, storage_params, force=False):
 552         indata.pop("onboardingState", None)
 553         indata.pop("operationalState", None)
 554         indata.pop("usageState", None)
 555         indata.pop("links", None)
 556
 557         indata = self.pyangbind_validation("vnfds", indata, force)
 558         # Cross references validation in the descriptor
 559
 560         self.validate_mgmt_interface_connection_point(indata)
 561
 562         for vdu in get_iterable(indata.get("vdu")):
 563             self.validate_vdu_internal_connection_points(vdu)
 564             self._validate_vdu_cloud_init_in_package(storage_params, vdu, indata)
 565         self._validate_vdu_charms_in_package(storage_params, indata)
 566
 567         self._validate_vnf_charms_in_package(storage_params, indata)
 568
 569         self.validate_external_connection_points(indata)
 570         self.validate_internal_virtual_links(indata)
 571         self.validate_monitoring_params(indata)
 572         self.validate_scaling_group_descriptor(indata)
 573
 574         return indata
 575
 576     @staticmethod
 577     def validate_mgmt_interface_connection_point(indata):
 578         if not indata.get("vdu"):
 579             return
 580         if not indata.get("mgmt-cp"):
 581             raise EngineException("'mgmt-cp' is a mandatory field and it is not defined",
 582                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 583
 584         for cp in get_iterable(indata.get("ext-cpd")):
 585             if cp["id"] == indata["mgmt-cp"]:
 586                 break
 587         else:
 588             raise EngineException("mgmt-cp='{}' must match an existing ext-cpd".format(indata["mgmt-cp"]),
 589                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 590
 591     @staticmethod
 592     def validate_vdu_internal_connection_points(vdu):
 593         int_cpds = set()
 594         for cpd in get_iterable(vdu.get("int-cpd")):
 595             cpd_id = cpd.get("id")
 596             if cpd_id and cpd_id in int_cpds:
 597                 raise EngineException("vdu[id='{}']:int-cpd[id='{}'] is already used by other int-cpd"
 598                                       .format(vdu["id"], cpd_id),
 599                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 600             int_cpds.add(cpd_id)
 601
 602     @staticmethod
 603     def validate_external_connection_points(indata):
 604         all_vdus_int_cpds = set()
 605         for vdu in get_iterable(indata.get("vdu")):
 606             for int_cpd in get_iterable(vdu.get("int-cpd")):
 607                 all_vdus_int_cpds.add((vdu.get("id"), int_cpd.get("id")))
 608
 609         ext_cpds = set()
 610         for cpd in get_iterable(indata.get("ext-cpd")):
 611             cpd_id = cpd.get("id")
 612             if cpd_id and cpd_id in ext_cpds:
 613                 raise EngineException("ext-cpd[id='{}'] is already used by other ext-cpd".format(cpd_id),
 614                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 615             ext_cpds.add(cpd_id)
 616
 617             int_cpd = cpd.get("int-cpd")
 618             if int_cpd:
 619                 if (int_cpd.get("vdu-id"), int_cpd.get("cpd")) not in all_vdus_int_cpds:
 620                     raise EngineException("ext-cpd[id='{}']:int-cpd must match an existing vdu int-cpd".format(cpd_id),
 621                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 622             # TODO: Validate k8s-cluster-net points to a valid k8s-cluster:nets ?
 623
 624     def _validate_vdu_charms_in_package(self, storage_params, indata):
 625         for df in indata["df"]:
 626             if "lcm-operations-configuration" in df and "operate-vnf-op-config" in df["lcm-operations-configuration"]:
 627                 configs = df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", [])
 628                 vdus = df.get("vdu-profile", [])
 629                 for vdu in vdus:
 630                     for config in configs:
 631                         if config["id"] == vdu["id"] and utils.find_in_list(
 632                             config.get("execution-environment-list", []),
 633                             lambda ee: "juju" in ee
 634                         ):
 635                             if not self._validate_package_folders(storage_params, 'charms'):
 636                                 raise EngineException("Charm defined in vnf[id={}] but not present in "
 637                                                       "package".format(indata["id"]))
 638
 639     def _validate_vdu_cloud_init_in_package(self, storage_params, vdu, indata):
 640         if not vdu.get("cloud-init-file"):
 641             return
 642         if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 643             raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 644                                   "package".format(indata["id"], vdu["id"]))
 645
 646     def _validate_vnf_charms_in_package(self, storage_params, indata):
 647         # Get VNF configuration through new container
 648         for deployment_flavor in indata.get('df', []):
 649             if "lcm-operations-configuration" not in deployment_flavor:
 650                 return
 651             if "operate-vnf-op-config" not in deployment_flavor["lcm-operations-configuration"]:
 652                 return
 653             for day_1_2_config in deployment_flavor["lcm-operations-configuration"]["operate-vnf-op-config"]["day1-2"]:
 654                 if day_1_2_config["id"] == indata["id"]:
 655                     if utils.find_in_list(
 656                         day_1_2_config.get("execution-environment-list", []),
 657                         lambda ee: "juju" in ee
 658                     ):
 659                         if not self._validate_package_folders(storage_params, 'charms'):
 660                             raise EngineException("Charm defined in vnf[id={}] but not present in "
 661                                                   "package".format(indata["id"]))
 662
 663     def _validate_package_folders(self, storage_params, folder, file=None):
 664         if not storage_params or not storage_params.get("pkg-dir"):
 665             return False
 666         else:
 667             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 668                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 669             else:
 670                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 671             if file:
 672                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 673             else:
 674                 if self.fs.file_exists(f, 'dir'):
 675                     if self.fs.dir_ls(f):
 676                         return True
 677             return False
 678
 679     @staticmethod
 680     def validate_internal_virtual_links(indata):
 681         all_ivld_ids = set()
 682         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 683             ivld_id = ivld.get("id")
 684             if ivld_id and ivld_id in all_ivld_ids:
 685                 raise EngineException("Duplicated VLD id in int-virtual-link-desc[id={}]".format(ivld_id),
 686                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 687             else:
 688                 all_ivld_ids.add(ivld_id)
 689
 690         for vdu in get_iterable(indata.get("vdu")):
 691             for int_cpd in get_iterable(vdu.get("int-cpd")):
 692                 int_cpd_ivld_id = int_cpd.get("int-virtual-link-desc")
 693                 if int_cpd_ivld_id and int_cpd_ivld_id not in all_ivld_ids:
 694                     raise EngineException(
 695                         "vdu[id='{}']:int-cpd[id='{}']:int-virtual-link-desc='{}' must match an existing "
 696                         "int-virtual-link-desc".format(vdu["id"], int_cpd["id"], int_cpd_ivld_id),
 697                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 698
 699         for df in get_iterable(indata.get("df")):
 700             for vlp in get_iterable(df.get("virtual-link-profile")):
 701                 vlp_ivld_id = vlp.get("id")
 702                 if vlp_ivld_id and vlp_ivld_id not in all_ivld_ids:
 703                     raise EngineException("df[id='{}']:virtual-link-profile='{}' must match an existing "
 704                                           "int-virtual-link-desc".format(df["id"], vlp_ivld_id),
 705                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 706
 707     @staticmethod
 708     def validate_monitoring_params(indata):
 709         all_monitoring_params = set()
 710         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 711             for mp in get_iterable(ivld.get("monitoring-parameters")):
 712                 mp_id = mp.get("id")
 713                 if mp_id and mp_id in all_monitoring_params:
 714                     raise EngineException("Duplicated monitoring-parameter id in "
 715                                           "int-virtual-link-desc[id='{}']:monitoring-parameters[id='{}']"
 716                                           .format(ivld["id"], mp_id),
 717                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 718                 else:
 719                     all_monitoring_params.add(mp_id)
 720
 721         for vdu in get_iterable(indata.get("vdu")):
 722             for mp in get_iterable(vdu.get("monitoring-parameter")):
 723                 mp_id = mp.get("id")
 724                 if mp_id and mp_id in all_monitoring_params:
 725                     raise EngineException("Duplicated monitoring-parameter id in "
 726                                           "vdu[id='{}']:monitoring-parameter[id='{}']"
 727                                           .format(vdu["id"], mp_id),
 728                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 729                 else:
 730                     all_monitoring_params.add(mp_id)
 731
 732         for df in get_iterable(indata.get("df")):
 733             for mp in get_iterable(df.get("monitoring-parameter")):
 734                 mp_id = mp.get("id")
 735                 if mp_id and mp_id in all_monitoring_params:
 736                     raise EngineException("Duplicated monitoring-parameter id in "
 737                                           "df[id='{}']:monitoring-parameter[id='{}']"
 738                                           .format(df["id"], mp_id),
 739                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 740                 else:
 741                     all_monitoring_params.add(mp_id)
 742
 743     @staticmethod
 744     def validate_scaling_group_descriptor(indata):
 745         all_monitoring_params = set()
 746         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 747             for mp in get_iterable(ivld.get("monitoring-parameters")):
 748                 all_monitoring_params.add(mp.get("id"))
 749
 750         for vdu in get_iterable(indata.get("vdu")):
 751             for mp in get_iterable(vdu.get("monitoring-parameter")):
 752                 all_monitoring_params.add(mp.get("id"))
 753
 754         for df in get_iterable(indata.get("df")):
 755             for mp in get_iterable(df.get("monitoring-parameter")):
 756                 all_monitoring_params.add(mp.get("id"))
 757
 758         for df in get_iterable(indata.get("df")):
 759             for sa in get_iterable(df.get("scaling-aspect")):
 760                 for sp in get_iterable(sa.get("scaling-policy")):
 761                     for sc in get_iterable(sp.get("scaling-criteria")):
 762                         sc_monitoring_param = sc.get("vnf-monitoring-param-ref")
 763                         if sc_monitoring_param and sc_monitoring_param not in all_monitoring_params:
 764                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-policy"
 765                                                   "[name='{}']:scaling-criteria[name='{}']: "
 766                                                   "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 767                                                   .format(df["id"], sa["id"], sp["name"], sc["name"],
 768                                                           sc_monitoring_param),
 769                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 770
 771                 for sca in get_iterable(sa.get("scaling-config-action")):
 772                     if "lcm-operations-configuration" not in df \
 773                         or "operate-vnf-op-config" not in df["lcm-operations-configuration"] \
 774                         or not utils.find_in_list(
 775                             df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", []),
 776                             lambda config: config["id"] == indata["id"]):
 777                         raise EngineException("'day1-2 configuration' not defined in the descriptor but it is "
 778                                               "referenced by df[id='{}']:scaling-aspect[id='{}']:scaling-config-action"
 779                                               .format(df["id"], sa["id"]),
 780                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 781                     for configuration in get_iterable(
 782                         df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", [])
 783                     ):
 784                         for primitive in get_iterable(configuration.get("config-primitive")):
 785                             if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 786                                 break
 787                         else:
 788                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-config-action:vnf-"
 789                                                   "config-primitive-name-ref='{}' does not match any "
 790                                                   "day1-2 configuration:config-primitive:name"
 791                                                   .format(df["id"], sa["id"], sca["vnf-config-primitive-name-ref"]),
 792                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 793
 794     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 795         """
 796         Deletes associate file system storage (via super)
 797         Deletes associated vnfpkgops from database.
 798         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 799         :param _id: server internal id
 800         :param db_content: The database content of the descriptor
 801         :return: None
 802         :raises: FsException in case of error while deleting associated storage
 803         """
 804         super().delete_extra(session, _id, db_content, not_send_msg)
 805         self.db.del_list("vnfpkgops", {"vnfPkgId": _id})
 806
 807     def sol005_projection(self, data):
 808         data["onboardingState"] = data["_admin"]["onboardingState"]
 809         data["operationalState"] = data["_admin"]["operationalState"]
 810         data["usageState"] = data["_admin"]["usageState"]
 811
 812         links = {}
 813         links["self"] = {"href": "/vnfpkgm/v1/vnf_packages/{}".format(data["_id"])}
 814         links["vnfd"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(data["_id"])}
 815         links["packageContent"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/package_content".format(data["_id"])}
 816         data["_links"] = links
 817
 818         return super().sol005_projection(data)
 819
 820
 821 class NsdTopic(DescriptorTopic):
 822     topic = "nsds"
 823     topic_msg = "nsd"
 824
 825     def __init__(self, db, fs, msg, auth):
 826         DescriptorTopic.__init__(self, db, fs, msg, auth)
 827
 828     def pyangbind_validation(self, item, data, force=False):
 829         if self._descriptor_data_is_in_old_format(data):
 830             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 831                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 832         try:
 833             nsd_vnf_profiles = data.get('df', [{}])[0].get('vnf-profile', [])
 834             mynsd = etsi_nfv_nsd.etsi_nfv_nsd()
 835             pybindJSONDecoder.load_ietf_json({'nsd': {'nsd': [data]}}, None, None, obj=mynsd,
 836                                              path_helper=True, skip_unknown=force)
 837             out = pybindJSON.dumps(mynsd, mode="ietf")
 838             desc_out = self._remove_envelop(yaml.safe_load(out))
 839             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 840             if nsd_vnf_profiles:
 841                 desc_out['df'][0]['vnf-profile'] = nsd_vnf_profiles
 842             return desc_out
 843         except Exception as e:
 844             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 845                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 846
 847     @staticmethod
 848     def _descriptor_data_is_in_old_format(data):
 849         return ('nsd-catalog' in data) or ('nsd:nsd-catalog' in data)
 850
 851     @staticmethod
 852     def _remove_envelop(indata=None):
 853         if not indata:
 854             return {}
 855         clean_indata = indata
 856
 857         if clean_indata.get('nsd'):
 858             clean_indata = clean_indata['nsd']
 859         elif clean_indata.get('etsi-nfv-nsd:nsd'):
 860             clean_indata = clean_indata['etsi-nfv-nsd:nsd']
 861         if clean_indata.get('nsd'):
 862             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 863                 raise EngineException("'nsd' must be a list of only one element")
 864             clean_indata = clean_indata['nsd'][0]
 865         return clean_indata
 866
 867     def _validate_input_new(self, indata, storage_params, force=False):
 868         indata.pop("nsdOnboardingState", None)
 869         indata.pop("nsdOperationalState", None)
 870         indata.pop("nsdUsageState", None)
 871
 872         indata.pop("links", None)
 873
 874         indata = self.pyangbind_validation("nsds", indata, force)
 875         # Cross references validation in the descriptor
 876         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 877         for vld in get_iterable(indata.get("virtual-link-desc")):
 878             self.validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata)
 879
 880         self.validate_vnf_profiles_vnfd_id(indata)
 881
 882         return indata
 883
 884     @staticmethod
 885     def validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata):
 886         if not vld.get("mgmt-network"):
 887             return
 888         vld_id = vld.get("id")
 889         for df in get_iterable(indata.get("df")):
 890             for vlp in get_iterable(df.get("virtual-link-profile")):
 891                 if vld_id and vld_id == vlp.get("virtual-link-desc-id"):
 892                     if vlp.get("virtual-link-protocol-data"):
 893                         raise EngineException("Error at df[id='{}']:virtual-link-profile[id='{}']:virtual-link-"
 894                                               "protocol-data You cannot set a virtual-link-protocol-data "
 895                                               "when mgmt-network is True"
 896                                               .format(df["id"], vlp["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 897
 898     @staticmethod
 899     def validate_vnf_profiles_vnfd_id(indata):
 900         all_vnfd_ids = set(get_iterable(indata.get("vnfd-id")))
 901         for df in get_iterable(indata.get("df")):
 902             for vnf_profile in get_iterable(df.get("vnf-profile")):
 903                 vnfd_id = vnf_profile.get("vnfd-id")
 904                 if vnfd_id and vnfd_id not in all_vnfd_ids:
 905                     raise EngineException("Error at df[id='{}']:vnf_profile[id='{}']:vnfd-id='{}' "
 906                                           "does not match any vnfd-id".format(df["id"], vnf_profile["id"], vnfd_id),
 907                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 908
 909     def _validate_input_edit(self, indata, content, force=False):
 910         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 911         """
 912         indata looks as follows:
 913             - In the new case (conformant)
 914                 {'nsdOperationalState': 'DISABLED', 'userDefinedData': {'id': 'string23',
 915                 '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}}
 916             - In the old case (backwards-compatible)
 917                 {'id': 'string23', '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}
 918         """
 919         if "_admin" not in indata:
 920             indata["_admin"] = {}
 921
 922         if "nsdOperationalState" in indata:
 923             if indata["nsdOperationalState"] in ("ENABLED", "DISABLED"):
 924                 indata["_admin"]["operationalState"] = indata.pop("nsdOperationalState")
 925             else:
 926                 raise EngineException("State '{}' is not a valid operational state"
 927                                       .format(indata["nsdOperationalState"]),
 928                                       http_code=HTTPStatus.BAD_REQUEST)
 929
 930         # In the case of user defined data, we need to put the data in the root of the object
 931         # to preserve current expected behaviour
 932         if "userDefinedData" in indata:
 933             data = indata.pop("userDefinedData")
 934             if type(data) == dict:
 935                 indata["_admin"]["userDefinedData"] = data
 936             else:
 937                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 938                                       .format(type(data)),
 939                                       http_code=HTTPStatus.BAD_REQUEST)
 940         if ("operationalState" in indata["_admin"] and
 941                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 942             raise EngineException("nsdOperationalState already {}".format(content["_admin"]["operationalState"]),
 943                                   http_code=HTTPStatus.CONFLICT)
 944         return indata
 945
 946     def _check_descriptor_dependencies(self, session, descriptor):
 947         """
 948         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 949         connection points are ok
 950         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 951         :param descriptor: descriptor to be inserted or edit
 952         :return: None or raises exception
 953         """
 954         if session["force"]:
 955             return
 956         vnfds_index = self._get_descriptor_constituent_vnfds_index(session, descriptor)
 957
 958         # Cross references validation in the descriptor and vnfd connection point validation
 959         for df in get_iterable(descriptor.get("df")):
 960             self.validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index)
 961
 962     def _get_descriptor_constituent_vnfds_index(self, session, descriptor):
 963         vnfds_index = {}
 964         if descriptor.get("vnfd-id") and not session["force"]:
 965             for vnfd_id in get_iterable(descriptor.get("vnfd-id")):
 966                 query_filter = self._get_project_filter(session)
 967                 query_filter["id"] = vnfd_id
 968                 vnf_list = self.db.get_list("vnfds", query_filter)
 969                 if not vnf_list:
 970                     raise EngineException("Descriptor error at 'vnfd-id'='{}' references a non "
 971                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 972                 vnfds_index[vnfd_id] = vnf_list[0]
 973         return vnfds_index
 974
 975     @staticmethod
 976     def validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index):
 977         for vnf_profile in get_iterable(df.get("vnf-profile")):
 978             vnfd = vnfds_index.get(vnf_profile["vnfd-id"])
 979             all_vnfd_ext_cpds = set()
 980             for ext_cpd in get_iterable(vnfd.get("ext-cpd")):
 981                 if ext_cpd.get('id'):
 982                     all_vnfd_ext_cpds.add(ext_cpd.get('id'))
 983
 984             for virtual_link in get_iterable(vnf_profile.get("virtual-link-connectivity")):
 985                 for vl_cpd in get_iterable(virtual_link.get("constituent-cpd-id")):
 986                     vl_cpd_id = vl_cpd.get('constituent-cpd-id')
 987                     if vl_cpd_id and vl_cpd_id not in all_vnfd_ext_cpds:
 988                         raise EngineException("Error at df[id='{}']:vnf-profile[id='{}']:virtual-link-connectivity"
 989                                               "[virtual-link-profile-id='{}']:constituent-cpd-id='{}' references a "
 990                                               "non existing ext-cpd:id inside vnfd '{}'"
 991                                               .format(df["id"], vnf_profile["id"],
 992                                                       virtual_link["virtual-link-profile-id"], vl_cpd_id, vnfd["id"]),
 993                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 994
 995     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 996         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
 997
 998         self._check_descriptor_dependencies(session, final_content)
 999
1000         return final_content
1001
1002     def check_conflict_on_del(self, session, _id, db_content):
1003         """
1004         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
1005         that NSD can be public and be used by other projects.
1006         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1007         :param _id: nsd internal id
1008         :param db_content: The database content of the _id
1009         :return: None or raises EngineException with the conflict
1010         """
1011         if session["force"]:
1012             return
1013         descriptor = db_content
1014         descriptor_id = descriptor.get("id")
1015         if not descriptor_id:  # empty nsd not uploaded
1016             return
1017
1018         # check NSD used by NS
1019         _filter = self._get_project_filter(session)
1020         _filter["nsd-id"] = _id
1021         if self.db.get_list("nsrs", _filter):
1022             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
1023
1024         # check NSD referenced by NST
1025         del _filter["nsd-id"]
1026         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
1027         if self.db.get_list("nsts", _filter):
1028             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
1029                                   http_code=HTTPStatus.CONFLICT)
1030
1031     def sol005_projection(self, data):
1032         data["nsdOnboardingState"] = data["_admin"]["onboardingState"]
1033         data["nsdOperationalState"] = data["_admin"]["operationalState"]
1034         data["nsdUsageState"] = data["_admin"]["usageState"]
1035
1036         links = {}
1037         links["self"] = {"href": "/nsd/v1/ns_descriptors/{}".format(data["_id"])}
1038         links["nsd_content"] = {"href": "/nsd/v1/ns_descriptors/{}/nsd_content".format(data["_id"])}
1039         data["_links"] = links
1040
1041         return super().sol005_projection(data)
1042
1043
1044 class NstTopic(DescriptorTopic):
1045     topic = "nsts"
1046     topic_msg = "nst"
1047     quota_name = "slice_templates"
1048
1049     def __init__(self, db, fs, msg, auth):
1050         DescriptorTopic.__init__(self, db, fs, msg, auth)
1051
1052     def pyangbind_validation(self, item, data, force=False):
1053         try:
1054             mynst = nst_im()
1055             pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
1056                                              path_helper=True, skip_unknown=force)
1057             out = pybindJSON.dumps(mynst, mode="ietf")
1058             desc_out = self._remove_envelop(yaml.safe_load(out))
1059             return desc_out
1060         except Exception as e:
1061             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
1062                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
1063
1064     @staticmethod
1065     def _remove_envelop(indata=None):
1066         if not indata:
1067             return {}
1068         clean_indata = indata
1069
1070         if clean_indata.get('nst'):
1071             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
1072                 raise EngineException("'nst' must be a list only one element")
1073             clean_indata = clean_indata['nst'][0]
1074         elif clean_indata.get('nst:nst'):
1075             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
1076                 raise EngineException("'nst:nst' must be a list only one element")
1077             clean_indata = clean_indata['nst:nst'][0]
1078         return clean_indata
1079
1080     def _validate_input_new(self, indata, storage_params, force=False):
1081         indata.pop("onboardingState", None)
1082         indata.pop("operationalState", None)
1083         indata.pop("usageState", None)
1084         indata = self.pyangbind_validation("nsts", indata, force)
1085         return indata.copy()
1086
1087     def _check_descriptor_dependencies(self, session, descriptor):
1088         """
1089         Check that the dependent descriptors exist on a new descriptor or edition
1090         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1091         :param descriptor: descriptor to be inserted or edit
1092         :return: None or raises exception
1093         """
1094         if not descriptor.get("netslice-subnet"):
1095             return
1096         for nsd in descriptor["netslice-subnet"]:
1097             nsd_id = nsd["nsd-ref"]
1098             filter_q = self._get_project_filter(session)
1099             filter_q["id"] = nsd_id
1100             if not self.db.get_list("nsds", filter_q):
1101                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
1102                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
1103
1104     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
1105         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
1106
1107         self._check_descriptor_dependencies(session, final_content)
1108         return final_content
1109
1110     def check_conflict_on_del(self, session, _id, db_content):
1111         """
1112         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
1113         that NST can be public and be used by other projects.
1114         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1115         :param _id: nst internal id
1116         :param db_content: The database content of the _id.
1117         :return: None or raises EngineException with the conflict
1118         """
1119         # TODO: Check this method
1120         if session["force"]:
1121             return
1122         # Get Network Slice Template from Database
1123         _filter = self._get_project_filter(session)
1124         _filter["_admin.nst-id"] = _id
1125         if self.db.get_list("nsis", _filter):
1126             raise EngineException("there is at least one Netslice Instance using this descriptor",
1127                                   http_code=HTTPStatus.CONFLICT)
1128
1129     def sol005_projection(self, data):
1130         data["onboardingState"] = data["_admin"]["onboardingState"]
1131         data["operationalState"] = data["_admin"]["operationalState"]
1132         data["usageState"] = data["_admin"]["usageState"]
1133
1134         links = {}
1135         links["self"] = {"href": "/nst/v1/netslice_templates/{}".format(data["_id"])}
1136         links["nst"] = {"href": "/nst/v1/netslice_templates/{}/nst".format(data["_id"])}
1137         data["_links"] = links
1138
1139         return super().sol005_projection(data)
1140
1141
1142 class PduTopic(BaseTopic):
1143     topic = "pdus"
1144     topic_msg = "pdu"
1145     quota_name = "pduds"
1146     schema_new = pdu_new_schema
1147     schema_edit = pdu_edit_schema
1148
1149     def __init__(self, db, fs, msg, auth):
1150         BaseTopic.__init__(self, db, fs, msg, auth)
1151
1152     @staticmethod
1153     def format_on_new(content, project_id=None, make_public=False):
1154         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
1155         content["_admin"]["onboardingState"] = "CREATED"
1156         content["_admin"]["operationalState"] = "ENABLED"
1157         content["_admin"]["usageState"] = "NOT_IN_USE"
1158
1159     def check_conflict_on_del(self, session, _id, db_content):
1160         """
1161         Check that there is not any vnfr that uses this PDU
1162         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1163         :param _id: pdu internal id
1164         :param db_content: The database content of the _id.
1165         :return: None or raises EngineException with the conflict
1166         """
1167         if session["force"]:
1168             return
1169
1170         _filter = self._get_project_filter(session)
1171         _filter["vdur.pdu-id"] = _id
1172         if self.db.get_list("vnfrs", _filter):
1173             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)
1174
1175
1176 class VnfPkgOpTopic(BaseTopic):
1177     topic = "vnfpkgops"
1178     topic_msg = "vnfd"
1179     schema_new = vnfpkgop_new_schema
1180     schema_edit = None
1181
1182     def __init__(self, db, fs, msg, auth):
1183         BaseTopic.__init__(self, db, fs, msg, auth)
1184
1185     def edit(self, session, _id, indata=None, kwargs=None, content=None):
1186         raise EngineException("Method 'edit' not allowed for topic '{}'".format(self.topic),
1187                               HTTPStatus.METHOD_NOT_ALLOWED)
1188
1189     def delete(self, session, _id, dry_run=False):
1190         raise EngineException("Method 'delete' not allowed for topic '{}'".format(self.topic),
1191                               HTTPStatus.METHOD_NOT_ALLOWED)
1192
1193     def delete_list(self, session, filter_q=None):
1194         raise EngineException("Method 'delete_list' not allowed for topic '{}'".format(self.topic),
1195                               HTTPStatus.METHOD_NOT_ALLOWED)
1196
1197     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
1198         """
1199         Creates a new entry into database.
1200         :param rollback: list to append created items at database in case a rollback may to be done
1201         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1202         :param indata: data to be inserted
1203         :param kwargs: used to override the indata descriptor
1204         :param headers: http request headers
1205         :return: _id, op_id:
1206             _id: identity of the inserted data.
1207              op_id: None
1208         """
1209         self._update_input_with_kwargs(indata, kwargs)
1210         validate_input(indata, self.schema_new)
1211         vnfpkg_id = indata["vnfPkgId"]
1212         filter_q = BaseTopic._get_project_filter(session)
1213         filter_q["_id"] = vnfpkg_id
1214         vnfd = self.db.get_one("vnfds", filter_q)
1215         operation = indata["lcmOperationType"]
1216         kdu_name = indata["kdu_name"]
1217         for kdu in vnfd.get("kdu", []):
1218             if kdu["name"] == kdu_name:
1219                 helm_chart = kdu.get("helm-chart")
1220                 juju_bundle = kdu.get("juju-bundle")
1221                 break
1222         else:
1223             raise EngineException("Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id, kdu_name))
1224         if helm_chart:
1225             indata["helm-chart"] = helm_chart
1226             match = fullmatch(r"([^/]*)/([^/]*)", helm_chart)
1227             repo_name = match.group(1) if match else None
1228         elif juju_bundle:
1229             indata["juju-bundle"] = juju_bundle
1230             match = fullmatch(r"([^/]*)/([^/]*)", juju_bundle)
1231             repo_name = match.group(1) if match else None
1232         else:
1233             raise EngineException("Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']"
1234                                   .format(vnfpkg_id, kdu_name))
1235         if repo_name:
1236             del filter_q["_id"]
1237             filter_q["name"] = repo_name
1238             repo = self.db.get_one("k8srepos", filter_q)
1239             k8srepo_id = repo.get("_id")
1240             k8srepo_url = repo.get("url")
1241         else:
1242             k8srepo_id = None
1243             k8srepo_url = None
1244         indata["k8srepoId"] = k8srepo_id
1245         indata["k8srepo_url"] = k8srepo_url
1246         vnfpkgop_id = str(uuid4())
1247         vnfpkgop_desc = {
1248             "_id": vnfpkgop_id,
1249             "operationState": "PROCESSING",
1250             "vnfPkgId": vnfpkg_id,
1251             "lcmOperationType": operation,
1252             "isAutomaticInvocation": False,
1253             "isCancelPending": False,
1254             "operationParams": indata,
1255             "links": {
1256                 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id,
1257                 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id,
1258             }
1259         }
1260         self.format_on_new(vnfpkgop_desc, session["project_id"], make_public=session["public"])
1261         ctime = vnfpkgop_desc["_admin"]["created"]
1262         vnfpkgop_desc["statusEnteredTime"] = ctime
1263         vnfpkgop_desc["startTime"] = ctime
1264         self.db.create(self.topic, vnfpkgop_desc)
1265         rollback.append({"topic": self.topic, "_id": vnfpkgop_id})
1266         self.msg.write(self.topic_msg, operation, vnfpkgop_desc)
1267         return vnfpkgop_id, None