osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 import importlib
  20 import copy
  21 # import logging
  22 from hashlib import md5
  23 from osm_common.dbbase import DbException, deep_update_rfc7396
  24 from http import HTTPStatus
  25 from time import time
  26 from uuid import uuid4
  27 from re import fullmatch
  28 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema, \
  29     validate_input, vnfpkgop_new_schema
  30 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  31 etsi_nfv_vnfd = importlib.import_module("osm_im.etsi-nfv-vnfd")
  32 etsi_nfv_nsd = importlib.import_module("osm_im.etsi-nfv-nsd")
  33 from osm_im.nst import nst as nst_im
  34 from pyangbind.lib.serialise import pybindJSONDecoder
  35 import pyangbind.lib.pybindJSON as pybindJSON
  36 from osm_nbi import utils
  37
  38 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  39
  40
  41 class DescriptorTopic(BaseTopic):
  42
  43     def __init__(self, db, fs, msg, auth):
  44         BaseTopic.__init__(self, db, fs, msg, auth)
  45
  46     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  47         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
  48
  49         def _check_unique_id_name(descriptor, position=""):
  50             for desc_key, desc_item in descriptor.items():
  51                 if isinstance(desc_item, list) and desc_item:
  52                     used_ids = []
  53                     desc_item_id = None
  54                     for index, list_item in enumerate(desc_item):
  55                         if isinstance(list_item, dict):
  56                             _check_unique_id_name(list_item, "{}.{}[{}]"
  57                                                   .format(position, desc_key, index))
  58                             # Base case
  59                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  60                                 desc_item_id = "id" if list_item.get("id") else "name"
  61                             if desc_item_id and list_item.get(desc_item_id):
  62                                 if list_item[desc_item_id] in used_ids:
  63                                     position = "{}.{}[{}]".format(position, desc_key, index)
  64                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  65                                                           .format(desc_item_id, list_item[desc_item_id],
  66                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  67                                 used_ids.append(list_item[desc_item_id])
  68
  69         _check_unique_id_name(final_content)
  70         # 1. validate again with pyangbind
  71         # 1.1. remove internal keys
  72         internal_keys = {}
  73         for k in ("_id", "_admin"):
  74             if k in final_content:
  75                 internal_keys[k] = final_content.pop(k)
  76         storage_params = internal_keys["_admin"].get("storage")
  77         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  78
  79         # 1.2. modify final_content with a serialized version
  80         final_content = copy.deepcopy(serialized)
  81         # 1.3. restore internal keys
  82         for k, v in internal_keys.items():
  83             final_content[k] = v
  84         if session["force"]:
  85             return final_content
  86
  87         # 2. check that this id is not present
  88         if "id" in edit_content:
  89             _filter = self._get_project_filter(session)
  90
  91             _filter["id"] = final_content["id"]
  92             _filter["_id.neq"] = _id
  93
  94             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  95                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  96                                                                                                final_content["id"]),
  97                                       HTTPStatus.CONFLICT)
  98
  99         return final_content
 100
 101     @staticmethod
 102     def format_on_new(content, project_id=None, make_public=False):
 103         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 104         content["_admin"]["onboardingState"] = "CREATED"
 105         content["_admin"]["operationalState"] = "DISABLED"
 106         content["_admin"]["usageState"] = "NOT_IN_USE"
 107
 108     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 109         """
 110         Deletes file system storage associated with the descriptor
 111         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 112         :param _id: server internal id
 113         :param db_content: The database content of the descriptor
 114         :param not_send_msg: To not send message (False) or store content (list) instead
 115         :return: None if ok or raises EngineException with the problem
 116         """
 117         self.fs.file_delete(_id, ignore_non_exist=True)
 118         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 119
 120     @staticmethod
 121     def get_one_by_id(db, session, topic, id):
 122         # find owned by this project
 123         _filter = BaseTopic._get_project_filter(session)
 124         _filter["id"] = id
 125         desc_list = db.get_list(topic, _filter)
 126         if len(desc_list) == 1:
 127             return desc_list[0]
 128         elif len(desc_list) > 1:
 129             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 130                               HTTPStatus.CONFLICT)
 131
 132         # not found any: try to find public
 133         _filter = BaseTopic._get_project_filter(session)
 134         _filter["id"] = id
 135         desc_list = db.get_list(topic, _filter)
 136         if not desc_list:
 137             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 138         elif len(desc_list) == 1:
 139             return desc_list[0]
 140         else:
 141             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 142                 topic[:-1], id), HTTPStatus.CONFLICT)
 143
 144     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 145         """
 146         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 147         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 148         (self.upload_content)
 149         :param rollback: list to append created items at database in case a rollback may to be done
 150         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 151         :param indata: data to be inserted
 152         :param kwargs: used to override the indata descriptor
 153         :param headers: http request headers
 154         :return: _id, None: identity of the inserted data; and None as there is not any operation
 155         """
 156
 157         # No needed to capture exceptions
 158         # Check Quota
 159         self.check_quota(session)
 160
 161         # _remove_envelop
 162         if indata:
 163             if "userDefinedData" in indata:
 164                 indata = indata['userDefinedData']
 165
 166         # Override descriptor with query string kwargs
 167         self._update_input_with_kwargs(indata, kwargs)
 168         # uncomment when this method is implemented.
 169         # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 170         # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 171
 172         content = {"_admin": {"userDefinedData": indata}}
 173         self.format_on_new(content, session["project_id"], make_public=session["public"])
 174         _id = self.db.create(self.topic, content)
 175         rollback.append({"topic": self.topic, "_id": _id})
 176         self._send_msg("created", {"_id": _id})
 177         return _id, None
 178
 179     def upload_content(self, session, _id, indata, kwargs, headers):
 180         """
 181         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 182         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 183         :param _id : the nsd,vnfd is already created, this is the id
 184         :param indata: http body request
 185         :param kwargs: user query string to override parameters. NOT USED
 186         :param headers:  http request headers
 187         :return: True if package is completely uploaded or False if partial content has been uploded
 188             Raise exception on error
 189         """
 190         # Check that _id exists and it is valid
 191         current_desc = self.show(session, _id)
 192
 193         content_range_text = headers.get("Content-Range")
 194         expected_md5 = headers.get("Content-File-MD5")
 195         compressed = None
 196         content_type = headers.get("Content-Type")
 197         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 198                 "application/zip" in content_type:
 199             compressed = "gzip"
 200         filename = headers.get("Content-Filename")
 201         if not filename:
 202             filename = "package.tar.gz" if compressed else "package"
 203         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 204         file_pkg = None
 205         error_text = ""
 206         try:
 207             if content_range_text:
 208                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 209                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 210                     raise IndexError()
 211                 start = int(content_range[1])
 212                 end = int(content_range[2]) + 1
 213                 total = int(content_range[3])
 214             else:
 215                 start = 0
 216             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 217
 218             if start:
 219                 if not self.fs.file_exists(temp_folder, 'dir'):
 220                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 221             else:
 222                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 223                 self.fs.mkdir(temp_folder)
 224
 225             storage = self.fs.get_params()
 226             storage["folder"] = _id
 227
 228             file_path = (temp_folder, filename)
 229             if self.fs.file_exists(file_path, 'file'):
 230                 file_size = self.fs.file_size(file_path)
 231             else:
 232                 file_size = 0
 233             if file_size != start:
 234                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 235                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 236             file_pkg = self.fs.file_open(file_path, 'a+b')
 237             if isinstance(indata, dict):
 238                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 239                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 240             else:
 241                 indata_len = 0
 242                 while True:
 243                     indata_text = indata.read(4096)
 244                     indata_len += len(indata_text)
 245                     if not indata_text:
 246                         break
 247                     file_pkg.write(indata_text)
 248             if content_range_text:
 249                 if indata_len != end - start:
 250                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 251                         start, end - 1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 252                 if end != total:
 253                     # TODO update to UPLOADING
 254                     return False
 255
 256             # PACKAGE UPLOADED
 257             if expected_md5:
 258                 file_pkg.seek(0, 0)
 259                 file_md5 = md5()
 260                 chunk_data = file_pkg.read(1024)
 261                 while chunk_data:
 262                     file_md5.update(chunk_data)
 263                     chunk_data = file_pkg.read(1024)
 264                 if expected_md5 != file_md5.hexdigest():
 265                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 266             file_pkg.seek(0, 0)
 267             if compressed == "gzip":
 268                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 269                 descriptor_file_name = None
 270                 for tarinfo in tar:
 271                     tarname = tarinfo.name
 272                     tarname_path = tarname.split("/")
 273                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 274                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 275                     if len(tarname_path) == 1 and not tarinfo.isdir():
 276                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 277                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 278                         storage["pkg-dir"] = tarname_path[0]
 279                         if len(tarname_path) == 2:
 280                             if descriptor_file_name:
 281                                 raise EngineException(
 282                                     "Found more than one descriptor file at package descriptor tar.gz")
 283                             descriptor_file_name = tarname
 284                 if not descriptor_file_name:
 285                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 286                 storage["descriptor"] = descriptor_file_name
 287                 storage["zipfile"] = filename
 288                 self.fs.file_extract(tar, temp_folder)
 289                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 290                     content = descriptor_file.read()
 291             else:
 292                 content = file_pkg.read()
 293                 storage["descriptor"] = descriptor_file_name = filename
 294
 295             if descriptor_file_name.endswith(".json"):
 296                 error_text = "Invalid json format "
 297                 indata = json.load(content)
 298             else:
 299                 error_text = "Invalid yaml format "
 300                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 301
 302             current_desc["_admin"]["storage"] = storage
 303             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 304             current_desc["_admin"]["operationalState"] = "ENABLED"
 305
 306             indata = self._remove_envelop(indata)
 307
 308             # Override descriptor with query string kwargs
 309             if kwargs:
 310                 self._update_input_with_kwargs(indata, kwargs)
 311
 312             deep_update_rfc7396(current_desc, indata)
 313             current_desc = self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 314             current_desc["_admin"]["modified"] = time()
 315             self.db.replace(self.topic, _id, current_desc)
 316             self.fs.dir_rename(temp_folder, _id)
 317
 318             indata["_id"] = _id
 319             self._send_msg("edited", indata)
 320
 321             # TODO if descriptor has changed because kwargs update content and remove cached zip
 322             # TODO if zip is not present creates one
 323             return True
 324
 325         except EngineException:
 326             raise
 327         except IndexError:
 328             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 329                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 330         except IOError as e:
 331             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 332         except tarfile.ReadError as e:
 333             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 334         except (ValueError, yaml.YAMLError) as e:
 335             raise EngineException(error_text + str(e))
 336         except ValidationError as e:
 337             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 338         finally:
 339             if file_pkg:
 340                 file_pkg.close()
 341
 342     def get_file(self, session, _id, path=None, accept_header=None):
 343         """
 344         Return the file content of a vnfd or nsd
 345         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 346         :param _id: Identity of the vnfd, nsd
 347         :param path: artifact path or "$DESCRIPTOR" or None
 348         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 349         :return: opened file plus Accept format or raises an exception
 350         """
 351         accept_text = accept_zip = False
 352         if accept_header:
 353             if 'text/plain' in accept_header or '*/*' in accept_header:
 354                 accept_text = True
 355             if 'application/zip' in accept_header or '*/*' in accept_header:
 356                 accept_zip = 'application/zip'
 357             elif 'application/gzip' in accept_header:
 358                 accept_zip = 'application/gzip'
 359
 360         if not accept_text and not accept_zip:
 361             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 362                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 363
 364         content = self.show(session, _id)
 365         if content["_admin"]["onboardingState"] != "ONBOARDED":
 366             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 367                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 368                                   http_code=HTTPStatus.CONFLICT)
 369         storage = content["_admin"]["storage"]
 370         if path is not None and path != "$DESCRIPTOR":  # artifacts
 371             if not storage.get('pkg-dir'):
 372                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 373             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 374                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 375                 return folder_content, "text/plain"
 376                 # TODO manage folders in http
 377             else:
 378                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \
 379                     "application/octet-stream"
 380
 381         # pkgtype   accept  ZIP  TEXT    -> result
 382         # manyfiles         yes  X       -> zip
 383         #                   no   yes     -> error
 384         # onefile           yes  no      -> zip
 385         #                   X    yes     -> text
 386         contain_many_files = False
 387         if storage.get('pkg-dir'):
 388             # check if there are more than one file in the package, ignoring checksums.txt.
 389             pkg_files = self.fs.dir_ls((storage['folder'], storage['pkg-dir']))
 390             if len(pkg_files) >= 3 or (len(pkg_files) == 2 and 'checksums.txt' not in pkg_files):
 391                 contain_many_files = True
 392         if accept_text and (not contain_many_files or path == "$DESCRIPTOR"):
 393             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 394         elif contain_many_files and not accept_zip:
 395             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 396                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 397         else:
 398             if not storage.get('zipfile'):
 399                 # TODO generate zipfile if not present
 400                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 401                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 402             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 403
 404     def _remove_yang_prefixes_from_descriptor(self, descriptor):
 405         new_descriptor = {}
 406         for k, v in descriptor.items():
 407             new_v = v
 408             if isinstance(v, dict):
 409                 new_v = self._remove_yang_prefixes_from_descriptor(v)
 410             elif isinstance(v, list):
 411                 new_v = list()
 412                 for x in v:
 413                     if isinstance(x, dict):
 414                         new_v.append(self._remove_yang_prefixes_from_descriptor(x))
 415                     else:
 416                         new_v.append(x)
 417             new_descriptor[k.split(':')[-1]] = new_v
 418         return new_descriptor
 419
 420     def pyangbind_validation(self, item, data, force=False):
 421         raise EngineException("Not possible to validate '{}' item".format(item),
 422                               http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 423
 424     def _validate_input_edit(self, indata, content, force=False):
 425         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 426         if "_id" in indata:
 427             indata.pop("_id")
 428         if "_admin" not in indata:
 429             indata["_admin"] = {}
 430
 431         if "operationalState" in indata:
 432             if indata["operationalState"] in ("ENABLED", "DISABLED"):
 433                 indata["_admin"]["operationalState"] = indata.pop("operationalState")
 434             else:
 435                 raise EngineException("State '{}' is not a valid operational state"
 436                                       .format(indata["operationalState"]),
 437                                       http_code=HTTPStatus.BAD_REQUEST)
 438
 439         # In the case of user defined data, we need to put the data in the root of the object
 440         # to preserve current expected behaviour
 441         if "userDefinedData" in indata:
 442             data = indata.pop("userDefinedData")
 443             if type(data) == dict:
 444                 indata["_admin"]["userDefinedData"] = data
 445             else:
 446                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 447                                       .format(type(data)),
 448                                       http_code=HTTPStatus.BAD_REQUEST)
 449
 450         if ("operationalState" in indata["_admin"] and
 451                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 452             raise EngineException("operationalState already {}".format(content["_admin"]["operationalState"]),
 453                                   http_code=HTTPStatus.CONFLICT)
 454
 455         return indata
 456
 457
 458 class VnfdTopic(DescriptorTopic):
 459     topic = "vnfds"
 460     topic_msg = "vnfd"
 461
 462     def __init__(self, db, fs, msg, auth):
 463         DescriptorTopic.__init__(self, db, fs, msg, auth)
 464
 465     def pyangbind_validation(self, item, data, force=False):
 466         if self._descriptor_data_is_in_old_format(data):
 467             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 468                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 469         try:
 470             myvnfd = etsi_nfv_vnfd.etsi_nfv_vnfd()
 471             pybindJSONDecoder.load_ietf_json({'etsi-nfv-vnfd:vnfd': data}, None, None, obj=myvnfd,
 472                                              path_helper=True, skip_unknown=force)
 473             out = pybindJSON.dumps(myvnfd, mode="ietf")
 474             desc_out = self._remove_envelop(yaml.safe_load(out))
 475             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 476             return utils.deep_update_dict(data, desc_out)
 477         except Exception as e:
 478             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 479                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 480
 481     @staticmethod
 482     def _descriptor_data_is_in_old_format(data):
 483         return ('vnfd-catalog' in data) or ('vnfd:vnfd-catalog' in data)
 484
 485     @staticmethod
 486     def _remove_envelop(indata=None):
 487         if not indata:
 488             return {}
 489         clean_indata = indata
 490
 491         if clean_indata.get('etsi-nfv-vnfd:vnfd'):
 492             if not isinstance(clean_indata['etsi-nfv-vnfd:vnfd'], dict):
 493                 raise EngineException("'etsi-nfv-vnfd:vnfd' must be a dict")
 494             clean_indata = clean_indata['etsi-nfv-vnfd:vnfd']
 495         elif clean_indata.get('vnfd'):
 496             if not isinstance(clean_indata['vnfd'], dict):
 497                 raise EngineException("'vnfd' must be dict")
 498             clean_indata = clean_indata['vnfd']
 499
 500         return clean_indata
 501
 502     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 503         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
 504
 505         # set type of vnfd
 506         contains_pdu = False
 507         contains_vdu = False
 508         for vdu in get_iterable(final_content.get("vdu")):
 509             if vdu.get("pdu-type"):
 510                 contains_pdu = True
 511             else:
 512                 contains_vdu = True
 513         if contains_pdu:
 514             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 515         elif contains_vdu:
 516             final_content["_admin"]["type"] = "vnfd"
 517         # if neither vud nor pdu do not fill type
 518         return final_content
 519
 520     def check_conflict_on_del(self, session, _id, db_content):
 521         """
 522         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 523         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 524         that uses this vnfd
 525         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 526         :param _id: vnfd internal id
 527         :param db_content: The database content of the _id.
 528         :return: None or raises EngineException with the conflict
 529         """
 530         if session["force"]:
 531             return
 532         descriptor = db_content
 533         descriptor_id = descriptor.get("id")
 534         if not descriptor_id:  # empty vnfd not uploaded
 535             return
 536
 537         _filter = self._get_project_filter(session)
 538
 539         # check vnfrs using this vnfd
 540         _filter["vnfd-id"] = _id
 541         if self.db.get_list("vnfrs", _filter):
 542             raise EngineException("There is at least one VNF instance using this descriptor",
 543                                   http_code=HTTPStatus.CONFLICT)
 544
 545         # check NSD referencing this VNFD
 546         del _filter["vnfd-id"]
 547         _filter["vnfd-id"] = descriptor_id
 548         if self.db.get_list("nsds", _filter):
 549             raise EngineException("There is at least one NS package referencing this descriptor",
 550                                   http_code=HTTPStatus.CONFLICT)
 551
 552     def _validate_input_new(self, indata, storage_params, force=False):
 553         indata.pop("onboardingState", None)
 554         indata.pop("operationalState", None)
 555         indata.pop("usageState", None)
 556         indata.pop("links", None)
 557
 558         indata = self.pyangbind_validation("vnfds", indata, force)
 559         # Cross references validation in the descriptor
 560
 561         self.validate_mgmt_interface_connection_point(indata)
 562
 563         for vdu in get_iterable(indata.get("vdu")):
 564             self.validate_vdu_internal_connection_points(vdu)
 565             self._validate_vdu_cloud_init_in_package(storage_params, vdu, indata)
 566         self._validate_vdu_charms_in_package(storage_params, indata)
 567
 568         self._validate_vnf_charms_in_package(storage_params, indata)
 569
 570         self.validate_external_connection_points(indata)
 571         self.validate_internal_virtual_links(indata)
 572         self.validate_monitoring_params(indata)
 573         self.validate_scaling_group_descriptor(indata)
 574
 575         return indata
 576
 577     @staticmethod
 578     def validate_mgmt_interface_connection_point(indata):
 579         if not indata.get("vdu"):
 580             return
 581         if not indata.get("mgmt-cp"):
 582             raise EngineException("'mgmt-cp' is a mandatory field and it is not defined",
 583                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 584
 585         for cp in get_iterable(indata.get("ext-cpd")):
 586             if cp["id"] == indata["mgmt-cp"]:
 587                 break
 588         else:
 589             raise EngineException("mgmt-cp='{}' must match an existing ext-cpd".format(indata["mgmt-cp"]),
 590                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 591
 592     @staticmethod
 593     def validate_vdu_internal_connection_points(vdu):
 594         int_cpds = set()
 595         for cpd in get_iterable(vdu.get("int-cpd")):
 596             cpd_id = cpd.get("id")
 597             if cpd_id and cpd_id in int_cpds:
 598                 raise EngineException("vdu[id='{}']:int-cpd[id='{}'] is already used by other int-cpd"
 599                                       .format(vdu["id"], cpd_id),
 600                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 601             int_cpds.add(cpd_id)
 602
 603     @staticmethod
 604     def validate_external_connection_points(indata):
 605         all_vdus_int_cpds = set()
 606         for vdu in get_iterable(indata.get("vdu")):
 607             for int_cpd in get_iterable(vdu.get("int-cpd")):
 608                 all_vdus_int_cpds.add((vdu.get("id"), int_cpd.get("id")))
 609
 610         ext_cpds = set()
 611         for cpd in get_iterable(indata.get("ext-cpd")):
 612             cpd_id = cpd.get("id")
 613             if cpd_id and cpd_id in ext_cpds:
 614                 raise EngineException("ext-cpd[id='{}'] is already used by other ext-cpd".format(cpd_id),
 615                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 616             ext_cpds.add(cpd_id)
 617
 618             int_cpd = cpd.get("int-cpd")
 619             if int_cpd:
 620                 if (int_cpd.get("vdu-id"), int_cpd.get("cpd")) not in all_vdus_int_cpds:
 621                     raise EngineException("ext-cpd[id='{}']:int-cpd must match an existing vdu int-cpd".format(cpd_id),
 622                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 623             # TODO: Validate k8s-cluster-net points to a valid k8s-cluster:nets ?
 624
 625     def _validate_vdu_charms_in_package(self, storage_params, indata):
 626         for df in indata["df"]:
 627             if "lcm-operations-configuration" in df and "operate-vnf-op-config" in df["lcm-operations-configuration"]:
 628                 configs = df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", [])
 629                 vdus = df.get("vdu-profile", [])
 630                 for vdu in vdus:
 631                     for config in configs:
 632                         if config["id"] == vdu["id"] and utils.find_in_list(
 633                             config.get("execution-environment-list", []),
 634                             lambda ee: "juju" in ee
 635                         ):
 636                             if not self._validate_package_folders(storage_params, 'charms'):
 637                                 raise EngineException("Charm defined in vnf[id={}] but not present in "
 638                                                       "package".format(indata["id"]))
 639
 640     def _validate_vdu_cloud_init_in_package(self, storage_params, vdu, indata):
 641         if not vdu.get("cloud-init-file"):
 642             return
 643         if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 644             raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 645                                   "package".format(indata["id"], vdu["id"]))
 646
 647     def _validate_vnf_charms_in_package(self, storage_params, indata):
 648         # Get VNF configuration through new container
 649         for deployment_flavor in indata.get('df', []):
 650             if "lcm-operations-configuration" not in deployment_flavor:
 651                 return
 652             if "operate-vnf-op-config" not in deployment_flavor["lcm-operations-configuration"]:
 653                 return
 654             for day_1_2_config in deployment_flavor["lcm-operations-configuration"]["operate-vnf-op-config"]["day1-2"]:
 655                 if day_1_2_config["id"] == indata["id"]:
 656                     if utils.find_in_list(
 657                         day_1_2_config.get("execution-environment-list", []),
 658                         lambda ee: "juju" in ee
 659                     ):
 660                         if not self._validate_package_folders(storage_params, 'charms'):
 661                             raise EngineException("Charm defined in vnf[id={}] but not present in "
 662                                                   "package".format(indata["id"]))
 663
 664     def _validate_package_folders(self, storage_params, folder, file=None):
 665         if not storage_params or not storage_params.get("pkg-dir"):
 666             return False
 667         else:
 668             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 669                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 670             else:
 671                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 672             if file:
 673                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 674             else:
 675                 if self.fs.file_exists(f, 'dir'):
 676                     if self.fs.dir_ls(f):
 677                         return True
 678             return False
 679
 680     @staticmethod
 681     def validate_internal_virtual_links(indata):
 682         all_ivld_ids = set()
 683         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 684             ivld_id = ivld.get("id")
 685             if ivld_id and ivld_id in all_ivld_ids:
 686                 raise EngineException("Duplicated VLD id in int-virtual-link-desc[id={}]".format(ivld_id),
 687                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 688             else:
 689                 all_ivld_ids.add(ivld_id)
 690
 691         for vdu in get_iterable(indata.get("vdu")):
 692             for int_cpd in get_iterable(vdu.get("int-cpd")):
 693                 int_cpd_ivld_id = int_cpd.get("int-virtual-link-desc")
 694                 if int_cpd_ivld_id and int_cpd_ivld_id not in all_ivld_ids:
 695                     raise EngineException(
 696                         "vdu[id='{}']:int-cpd[id='{}']:int-virtual-link-desc='{}' must match an existing "
 697                         "int-virtual-link-desc".format(vdu["id"], int_cpd["id"], int_cpd_ivld_id),
 698                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 699
 700         for df in get_iterable(indata.get("df")):
 701             for vlp in get_iterable(df.get("virtual-link-profile")):
 702                 vlp_ivld_id = vlp.get("id")
 703                 if vlp_ivld_id and vlp_ivld_id not in all_ivld_ids:
 704                     raise EngineException("df[id='{}']:virtual-link-profile='{}' must match an existing "
 705                                           "int-virtual-link-desc".format(df["id"], vlp_ivld_id),
 706                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 707
 708     @staticmethod
 709     def validate_monitoring_params(indata):
 710         all_monitoring_params = set()
 711         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 712             for mp in get_iterable(ivld.get("monitoring-parameters")):
 713                 mp_id = mp.get("id")
 714                 if mp_id and mp_id in all_monitoring_params:
 715                     raise EngineException("Duplicated monitoring-parameter id in "
 716                                           "int-virtual-link-desc[id='{}']:monitoring-parameters[id='{}']"
 717                                           .format(ivld["id"], mp_id),
 718                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 719                 else:
 720                     all_monitoring_params.add(mp_id)
 721
 722         for vdu in get_iterable(indata.get("vdu")):
 723             for mp in get_iterable(vdu.get("monitoring-parameter")):
 724                 mp_id = mp.get("id")
 725                 if mp_id and mp_id in all_monitoring_params:
 726                     raise EngineException("Duplicated monitoring-parameter id in "
 727                                           "vdu[id='{}']:monitoring-parameter[id='{}']"
 728                                           .format(vdu["id"], mp_id),
 729                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 730                 else:
 731                     all_monitoring_params.add(mp_id)
 732
 733         for df in get_iterable(indata.get("df")):
 734             for mp in get_iterable(df.get("monitoring-parameter")):
 735                 mp_id = mp.get("id")
 736                 if mp_id and mp_id in all_monitoring_params:
 737                     raise EngineException("Duplicated monitoring-parameter id in "
 738                                           "df[id='{}']:monitoring-parameter[id='{}']"
 739                                           .format(df["id"], mp_id),
 740                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 741                 else:
 742                     all_monitoring_params.add(mp_id)
 743
 744     @staticmethod
 745     def validate_scaling_group_descriptor(indata):
 746         all_monitoring_params = set()
 747         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 748             for mp in get_iterable(ivld.get("monitoring-parameters")):
 749                 all_monitoring_params.add(mp.get("id"))
 750
 751         for vdu in get_iterable(indata.get("vdu")):
 752             for mp in get_iterable(vdu.get("monitoring-parameter")):
 753                 all_monitoring_params.add(mp.get("id"))
 754
 755         for df in get_iterable(indata.get("df")):
 756             for mp in get_iterable(df.get("monitoring-parameter")):
 757                 all_monitoring_params.add(mp.get("id"))
 758
 759         for df in get_iterable(indata.get("df")):
 760             for sa in get_iterable(df.get("scaling-aspect")):
 761                 for sp in get_iterable(sa.get("scaling-policy")):
 762                     for sc in get_iterable(sp.get("scaling-criteria")):
 763                         sc_monitoring_param = sc.get("vnf-monitoring-param-ref")
 764                         if sc_monitoring_param and sc_monitoring_param not in all_monitoring_params:
 765                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-policy"
 766                                                   "[name='{}']:scaling-criteria[name='{}']: "
 767                                                   "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 768                                                   .format(df["id"], sa["id"], sp["name"], sc["name"],
 769                                                           sc_monitoring_param),
 770                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 771
 772                 for sca in get_iterable(sa.get("scaling-config-action")):
 773                     if "lcm-operations-configuration" not in df \
 774                         or "operate-vnf-op-config" not in df["lcm-operations-configuration"] \
 775                         or not utils.find_in_list(
 776                             df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", []),
 777                             lambda config: config["id"] == indata["id"]):
 778                         raise EngineException("'day1-2 configuration' not defined in the descriptor but it is "
 779                                               "referenced by df[id='{}']:scaling-aspect[id='{}']:scaling-config-action"
 780                                               .format(df["id"], sa["id"]),
 781                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 782                     for configuration in get_iterable(
 783                         df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", [])
 784                     ):
 785                         for primitive in get_iterable(configuration.get("config-primitive")):
 786                             if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 787                                 break
 788                         else:
 789                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-config-action:vnf-"
 790                                                   "config-primitive-name-ref='{}' does not match any "
 791                                                   "day1-2 configuration:config-primitive:name"
 792                                                   .format(df["id"], sa["id"], sca["vnf-config-primitive-name-ref"]),
 793                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 794
 795     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 796         """
 797         Deletes associate file system storage (via super)
 798         Deletes associated vnfpkgops from database.
 799         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 800         :param _id: server internal id
 801         :param db_content: The database content of the descriptor
 802         :return: None
 803         :raises: FsException in case of error while deleting associated storage
 804         """
 805         super().delete_extra(session, _id, db_content, not_send_msg)
 806         self.db.del_list("vnfpkgops", {"vnfPkgId": _id})
 807
 808     def sol005_projection(self, data):
 809         data["onboardingState"] = data["_admin"]["onboardingState"]
 810         data["operationalState"] = data["_admin"]["operationalState"]
 811         data["usageState"] = data["_admin"]["usageState"]
 812
 813         links = {}
 814         links["self"] = {"href": "/vnfpkgm/v1/vnf_packages/{}".format(data["_id"])}
 815         links["vnfd"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(data["_id"])}
 816         links["packageContent"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/package_content".format(data["_id"])}
 817         data["_links"] = links
 818
 819         return super().sol005_projection(data)
 820
 821
 822 class NsdTopic(DescriptorTopic):
 823     topic = "nsds"
 824     topic_msg = "nsd"
 825
 826     def __init__(self, db, fs, msg, auth):
 827         DescriptorTopic.__init__(self, db, fs, msg, auth)
 828
 829     def pyangbind_validation(self, item, data, force=False):
 830         if self._descriptor_data_is_in_old_format(data):
 831             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 832                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 833         try:
 834             nsd_vnf_profiles = data.get('df', [{}])[0].get('vnf-profile', [])
 835             mynsd = etsi_nfv_nsd.etsi_nfv_nsd()
 836             pybindJSONDecoder.load_ietf_json({'nsd': {'nsd': [data]}}, None, None, obj=mynsd,
 837                                              path_helper=True, skip_unknown=force)
 838             out = pybindJSON.dumps(mynsd, mode="ietf")
 839             desc_out = self._remove_envelop(yaml.safe_load(out))
 840             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 841             if nsd_vnf_profiles:
 842                 desc_out['df'][0]['vnf-profile'] = nsd_vnf_profiles
 843             return desc_out
 844         except Exception as e:
 845             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 846                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 847
 848     @staticmethod
 849     def _descriptor_data_is_in_old_format(data):
 850         return ('nsd-catalog' in data) or ('nsd:nsd-catalog' in data)
 851
 852     @staticmethod
 853     def _remove_envelop(indata=None):
 854         if not indata:
 855             return {}
 856         clean_indata = indata
 857
 858         if clean_indata.get('nsd'):
 859             clean_indata = clean_indata['nsd']
 860         elif clean_indata.get('etsi-nfv-nsd:nsd'):
 861             clean_indata = clean_indata['etsi-nfv-nsd:nsd']
 862         if clean_indata.get('nsd'):
 863             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 864                 raise EngineException("'nsd' must be a list of only one element")
 865             clean_indata = clean_indata['nsd'][0]
 866         return clean_indata
 867
 868     def _validate_input_new(self, indata, storage_params, force=False):
 869         indata.pop("nsdOnboardingState", None)
 870         indata.pop("nsdOperationalState", None)
 871         indata.pop("nsdUsageState", None)
 872
 873         indata.pop("links", None)
 874
 875         indata = self.pyangbind_validation("nsds", indata, force)
 876         # Cross references validation in the descriptor
 877         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 878         for vld in get_iterable(indata.get("virtual-link-desc")):
 879             self.validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata)
 880
 881         self.validate_vnf_profiles_vnfd_id(indata)
 882
 883         return indata
 884
 885     @staticmethod
 886     def validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata):
 887         if not vld.get("mgmt-network"):
 888             return
 889         vld_id = vld.get("id")
 890         for df in get_iterable(indata.get("df")):
 891             for vlp in get_iterable(df.get("virtual-link-profile")):
 892                 if vld_id and vld_id == vlp.get("virtual-link-desc-id"):
 893                     if vlp.get("virtual-link-protocol-data"):
 894                         raise EngineException("Error at df[id='{}']:virtual-link-profile[id='{}']:virtual-link-"
 895                                               "protocol-data You cannot set a virtual-link-protocol-data "
 896                                               "when mgmt-network is True"
 897                                               .format(df["id"], vlp["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 898
 899     @staticmethod
 900     def validate_vnf_profiles_vnfd_id(indata):
 901         all_vnfd_ids = set(get_iterable(indata.get("vnfd-id")))
 902         for df in get_iterable(indata.get("df")):
 903             for vnf_profile in get_iterable(df.get("vnf-profile")):
 904                 vnfd_id = vnf_profile.get("vnfd-id")
 905                 if vnfd_id and vnfd_id not in all_vnfd_ids:
 906                     raise EngineException("Error at df[id='{}']:vnf_profile[id='{}']:vnfd-id='{}' "
 907                                           "does not match any vnfd-id".format(df["id"], vnf_profile["id"], vnfd_id),
 908                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 909
 910     def _validate_input_edit(self, indata, content, force=False):
 911         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 912         """
 913         indata looks as follows:
 914             - In the new case (conformant)
 915                 {'nsdOperationalState': 'DISABLED', 'userDefinedData': {'id': 'string23',
 916                 '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}}
 917             - In the old case (backwards-compatible)
 918                 {'id': 'string23', '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}
 919         """
 920         if "_admin" not in indata:
 921             indata["_admin"] = {}
 922
 923         if "nsdOperationalState" in indata:
 924             if indata["nsdOperationalState"] in ("ENABLED", "DISABLED"):
 925                 indata["_admin"]["operationalState"] = indata.pop("nsdOperationalState")
 926             else:
 927                 raise EngineException("State '{}' is not a valid operational state"
 928                                       .format(indata["nsdOperationalState"]),
 929                                       http_code=HTTPStatus.BAD_REQUEST)
 930
 931         # In the case of user defined data, we need to put the data in the root of the object
 932         # to preserve current expected behaviour
 933         if "userDefinedData" in indata:
 934             data = indata.pop("userDefinedData")
 935             if type(data) == dict:
 936                 indata["_admin"]["userDefinedData"] = data
 937             else:
 938                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 939                                       .format(type(data)),
 940                                       http_code=HTTPStatus.BAD_REQUEST)
 941         if ("operationalState" in indata["_admin"] and
 942                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 943             raise EngineException("nsdOperationalState already {}".format(content["_admin"]["operationalState"]),
 944                                   http_code=HTTPStatus.CONFLICT)
 945         return indata
 946
 947     def _check_descriptor_dependencies(self, session, descriptor):
 948         """
 949         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 950         connection points are ok
 951         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 952         :param descriptor: descriptor to be inserted or edit
 953         :return: None or raises exception
 954         """
 955         if session["force"]:
 956             return
 957         vnfds_index = self._get_descriptor_constituent_vnfds_index(session, descriptor)
 958
 959         # Cross references validation in the descriptor and vnfd connection point validation
 960         for df in get_iterable(descriptor.get("df")):
 961             self.validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index)
 962
 963     def _get_descriptor_constituent_vnfds_index(self, session, descriptor):
 964         vnfds_index = {}
 965         if descriptor.get("vnfd-id") and not session["force"]:
 966             for vnfd_id in get_iterable(descriptor.get("vnfd-id")):
 967                 query_filter = self._get_project_filter(session)
 968                 query_filter["id"] = vnfd_id
 969                 vnf_list = self.db.get_list("vnfds", query_filter)
 970                 if not vnf_list:
 971                     raise EngineException("Descriptor error at 'vnfd-id'='{}' references a non "
 972                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 973                 vnfds_index[vnfd_id] = vnf_list[0]
 974         return vnfds_index
 975
 976     @staticmethod
 977     def validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index):
 978         for vnf_profile in get_iterable(df.get("vnf-profile")):
 979             vnfd = vnfds_index.get(vnf_profile["vnfd-id"])
 980             all_vnfd_ext_cpds = set()
 981             for ext_cpd in get_iterable(vnfd.get("ext-cpd")):
 982                 if ext_cpd.get('id'):
 983                     all_vnfd_ext_cpds.add(ext_cpd.get('id'))
 984
 985             for virtual_link in get_iterable(vnf_profile.get("virtual-link-connectivity")):
 986                 for vl_cpd in get_iterable(virtual_link.get("constituent-cpd-id")):
 987                     vl_cpd_id = vl_cpd.get('constituent-cpd-id')
 988                     if vl_cpd_id and vl_cpd_id not in all_vnfd_ext_cpds:
 989                         raise EngineException("Error at df[id='{}']:vnf-profile[id='{}']:virtual-link-connectivity"
 990                                               "[virtual-link-profile-id='{}']:constituent-cpd-id='{}' references a "
 991                                               "non existing ext-cpd:id inside vnfd '{}'"
 992                                               .format(df["id"], vnf_profile["id"],
 993                                                       virtual_link["virtual-link-profile-id"], vl_cpd_id, vnfd["id"]),
 994                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 995
 996     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 997         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
 998
 999         self._check_descriptor_dependencies(session, final_content)
1000
1001         return final_content
1002
1003     def check_conflict_on_del(self, session, _id, db_content):
1004         """
1005         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
1006         that NSD can be public and be used by other projects.
1007         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1008         :param _id: nsd internal id
1009         :param db_content: The database content of the _id
1010         :return: None or raises EngineException with the conflict
1011         """
1012         if session["force"]:
1013             return
1014         descriptor = db_content
1015         descriptor_id = descriptor.get("id")
1016         if not descriptor_id:  # empty nsd not uploaded
1017             return
1018
1019         # check NSD used by NS
1020         _filter = self._get_project_filter(session)
1021         _filter["nsd-id"] = _id
1022         if self.db.get_list("nsrs", _filter):
1023             raise EngineException("There is at least one NS instance using this descriptor",
1024                                   http_code=HTTPStatus.CONFLICT)
1025
1026         # check NSD referenced by NST
1027         del _filter["nsd-id"]
1028         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
1029         if self.db.get_list("nsts", _filter):
1030             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
1031                                   http_code=HTTPStatus.CONFLICT)
1032
1033     def sol005_projection(self, data):
1034         data["nsdOnboardingState"] = data["_admin"]["onboardingState"]
1035         data["nsdOperationalState"] = data["_admin"]["operationalState"]
1036         data["nsdUsageState"] = data["_admin"]["usageState"]
1037
1038         links = {}
1039         links["self"] = {"href": "/nsd/v1/ns_descriptors/{}".format(data["_id"])}
1040         links["nsd_content"] = {"href": "/nsd/v1/ns_descriptors/{}/nsd_content".format(data["_id"])}
1041         data["_links"] = links
1042
1043         return super().sol005_projection(data)
1044
1045
1046 class NstTopic(DescriptorTopic):
1047     topic = "nsts"
1048     topic_msg = "nst"
1049     quota_name = "slice_templates"
1050
1051     def __init__(self, db, fs, msg, auth):
1052         DescriptorTopic.__init__(self, db, fs, msg, auth)
1053
1054     def pyangbind_validation(self, item, data, force=False):
1055         try:
1056             mynst = nst_im()
1057             pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
1058                                              path_helper=True, skip_unknown=force)
1059             out = pybindJSON.dumps(mynst, mode="ietf")
1060             desc_out = self._remove_envelop(yaml.safe_load(out))
1061             return desc_out
1062         except Exception as e:
1063             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
1064                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
1065
1066     @staticmethod
1067     def _remove_envelop(indata=None):
1068         if not indata:
1069             return {}
1070         clean_indata = indata
1071
1072         if clean_indata.get('nst'):
1073             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
1074                 raise EngineException("'nst' must be a list only one element")
1075             clean_indata = clean_indata['nst'][0]
1076         elif clean_indata.get('nst:nst'):
1077             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
1078                 raise EngineException("'nst:nst' must be a list only one element")
1079             clean_indata = clean_indata['nst:nst'][0]
1080         return clean_indata
1081
1082     def _validate_input_new(self, indata, storage_params, force=False):
1083         indata.pop("onboardingState", None)
1084         indata.pop("operationalState", None)
1085         indata.pop("usageState", None)
1086         indata = self.pyangbind_validation("nsts", indata, force)
1087         return indata.copy()
1088
1089     def _check_descriptor_dependencies(self, session, descriptor):
1090         """
1091         Check that the dependent descriptors exist on a new descriptor or edition
1092         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1093         :param descriptor: descriptor to be inserted or edit
1094         :return: None or raises exception
1095         """
1096         if not descriptor.get("netslice-subnet"):
1097             return
1098         for nsd in descriptor["netslice-subnet"]:
1099             nsd_id = nsd["nsd-ref"]
1100             filter_q = self._get_project_filter(session)
1101             filter_q["id"] = nsd_id
1102             if not self.db.get_list("nsds", filter_q):
1103                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
1104                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
1105
1106     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
1107         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
1108
1109         self._check_descriptor_dependencies(session, final_content)
1110         return final_content
1111
1112     def check_conflict_on_del(self, session, _id, db_content):
1113         """
1114         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
1115         that NST can be public and be used by other projects.
1116         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1117         :param _id: nst internal id
1118         :param db_content: The database content of the _id.
1119         :return: None or raises EngineException with the conflict
1120         """
1121         # TODO: Check this method
1122         if session["force"]:
1123             return
1124         # Get Network Slice Template from Database
1125         _filter = self._get_project_filter(session)
1126         _filter["_admin.nst-id"] = _id
1127         if self.db.get_list("nsis", _filter):
1128             raise EngineException("there is at least one Netslice Instance using this descriptor",
1129                                   http_code=HTTPStatus.CONFLICT)
1130
1131     def sol005_projection(self, data):
1132         data["onboardingState"] = data["_admin"]["onboardingState"]
1133         data["operationalState"] = data["_admin"]["operationalState"]
1134         data["usageState"] = data["_admin"]["usageState"]
1135
1136         links = {}
1137         links["self"] = {"href": "/nst/v1/netslice_templates/{}".format(data["_id"])}
1138         links["nst"] = {"href": "/nst/v1/netslice_templates/{}/nst".format(data["_id"])}
1139         data["_links"] = links
1140
1141         return super().sol005_projection(data)
1142
1143
1144 class PduTopic(BaseTopic):
1145     topic = "pdus"
1146     topic_msg = "pdu"
1147     quota_name = "pduds"
1148     schema_new = pdu_new_schema
1149     schema_edit = pdu_edit_schema
1150
1151     def __init__(self, db, fs, msg, auth):
1152         BaseTopic.__init__(self, db, fs, msg, auth)
1153
1154     @staticmethod
1155     def format_on_new(content, project_id=None, make_public=False):
1156         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
1157         content["_admin"]["onboardingState"] = "CREATED"
1158         content["_admin"]["operationalState"] = "ENABLED"
1159         content["_admin"]["usageState"] = "NOT_IN_USE"
1160
1161     def check_conflict_on_del(self, session, _id, db_content):
1162         """
1163         Check that there is not any vnfr that uses this PDU
1164         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1165         :param _id: pdu internal id
1166         :param db_content: The database content of the _id.
1167         :return: None or raises EngineException with the conflict
1168         """
1169         if session["force"]:
1170             return
1171
1172         _filter = self._get_project_filter(session)
1173         _filter["vdur.pdu-id"] = _id
1174         if self.db.get_list("vnfrs", _filter):
1175             raise EngineException("There is at least one VNF instance using this PDU", http_code=HTTPStatus.CONFLICT)
1176
1177
1178 class VnfPkgOpTopic(BaseTopic):
1179     topic = "vnfpkgops"
1180     topic_msg = "vnfd"
1181     schema_new = vnfpkgop_new_schema
1182     schema_edit = None
1183
1184     def __init__(self, db, fs, msg, auth):
1185         BaseTopic.__init__(self, db, fs, msg, auth)
1186
1187     def edit(self, session, _id, indata=None, kwargs=None, content=None):
1188         raise EngineException("Method 'edit' not allowed for topic '{}'".format(self.topic),
1189                               HTTPStatus.METHOD_NOT_ALLOWED)
1190
1191     def delete(self, session, _id, dry_run=False):
1192         raise EngineException("Method 'delete' not allowed for topic '{}'".format(self.topic),
1193                               HTTPStatus.METHOD_NOT_ALLOWED)
1194
1195     def delete_list(self, session, filter_q=None):
1196         raise EngineException("Method 'delete_list' not allowed for topic '{}'".format(self.topic),
1197                               HTTPStatus.METHOD_NOT_ALLOWED)
1198
1199     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
1200         """
1201         Creates a new entry into database.
1202         :param rollback: list to append created items at database in case a rollback may to be done
1203         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1204         :param indata: data to be inserted
1205         :param kwargs: used to override the indata descriptor
1206         :param headers: http request headers
1207         :return: _id, op_id:
1208             _id: identity of the inserted data.
1209              op_id: None
1210         """
1211         self._update_input_with_kwargs(indata, kwargs)
1212         validate_input(indata, self.schema_new)
1213         vnfpkg_id = indata["vnfPkgId"]
1214         filter_q = BaseTopic._get_project_filter(session)
1215         filter_q["_id"] = vnfpkg_id
1216         vnfd = self.db.get_one("vnfds", filter_q)
1217         operation = indata["lcmOperationType"]
1218         kdu_name = indata["kdu_name"]
1219         for kdu in vnfd.get("kdu", []):
1220             if kdu["name"] == kdu_name:
1221                 helm_chart = kdu.get("helm-chart")
1222                 juju_bundle = kdu.get("juju-bundle")
1223                 break
1224         else:
1225             raise EngineException("Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id, kdu_name))
1226         if helm_chart:
1227             indata["helm-chart"] = helm_chart
1228             match = fullmatch(r"([^/]*)/([^/]*)", helm_chart)
1229             repo_name = match.group(1) if match else None
1230         elif juju_bundle:
1231             indata["juju-bundle"] = juju_bundle
1232             match = fullmatch(r"([^/]*)/([^/]*)", juju_bundle)
1233             repo_name = match.group(1) if match else None
1234         else:
1235             raise EngineException("Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']"
1236                                   .format(vnfpkg_id, kdu_name))
1237         if repo_name:
1238             del filter_q["_id"]
1239             filter_q["name"] = repo_name
1240             repo = self.db.get_one("k8srepos", filter_q)
1241             k8srepo_id = repo.get("_id")
1242             k8srepo_url = repo.get("url")
1243         else:
1244             k8srepo_id = None
1245             k8srepo_url = None
1246         indata["k8srepoId"] = k8srepo_id
1247         indata["k8srepo_url"] = k8srepo_url
1248         vnfpkgop_id = str(uuid4())
1249         vnfpkgop_desc = {
1250             "_id": vnfpkgop_id,
1251             "operationState": "PROCESSING",
1252             "vnfPkgId": vnfpkg_id,
1253             "lcmOperationType": operation,
1254             "isAutomaticInvocation": False,
1255             "isCancelPending": False,
1256             "operationParams": indata,
1257             "links": {
1258                 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id,
1259                 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id,
1260             }
1261         }
1262         self.format_on_new(vnfpkgop_desc, session["project_id"], make_public=session["public"])
1263         ctime = vnfpkgop_desc["_admin"]["created"]
1264         vnfpkgop_desc["statusEnteredTime"] = ctime
1265         vnfpkgop_desc["startTime"] = ctime
1266         self.db.create(self.topic, vnfpkgop_desc)
1267         rollback.append({"topic": self.topic, "_id": vnfpkgop_id})
1268         self.msg.write(self.topic_msg, operation, vnfpkgop_desc)
1269         return vnfpkgop_id, None