osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 import importlib
  20 import copy
  21 # import logging
  22 from hashlib import md5
  23 from osm_common.dbbase import DbException, deep_update_rfc7396
  24 from http import HTTPStatus
  25 from time import time
  26 from uuid import uuid4
  27 from re import fullmatch
  28 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema, \
  29     validate_input, vnfpkgop_new_schema
  30 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  31 from osm_im import etsi_nfv_vnfd, etsi_nfv_nsd
  32 from osm_im.nst import nst as nst_im
  33 from pyangbind.lib.serialise import pybindJSONDecoder
  34 import pyangbind.lib.pybindJSON as pybindJSON
  35 from osm_nbi import utils
  36
  37 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  38
  39
  40 class DescriptorTopic(BaseTopic):
  41
  42     def __init__(self, db, fs, msg, auth):
  43         BaseTopic.__init__(self, db, fs, msg, auth)
  44
  45     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  46         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
  47
  48         def _check_unique_id_name(descriptor, position=""):
  49             for desc_key, desc_item in descriptor.items():
  50                 if isinstance(desc_item, list) and desc_item:
  51                     used_ids = []
  52                     desc_item_id = None
  53                     for index, list_item in enumerate(desc_item):
  54                         if isinstance(list_item, dict):
  55                             _check_unique_id_name(list_item, "{}.{}[{}]"
  56                                                   .format(position, desc_key, index))
  57                             # Base case
  58                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  59                                 desc_item_id = "id" if list_item.get("id") else "name"
  60                             if desc_item_id and list_item.get(desc_item_id):
  61                                 if list_item[desc_item_id] in used_ids:
  62                                     position = "{}.{}[{}]".format(position, desc_key, index)
  63                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  64                                                           .format(desc_item_id, list_item[desc_item_id],
  65                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  66                                 used_ids.append(list_item[desc_item_id])
  67
  68         _check_unique_id_name(final_content)
  69         # 1. validate again with pyangbind
  70         # 1.1. remove internal keys
  71         internal_keys = {}
  72         for k in ("_id", "_admin"):
  73             if k in final_content:
  74                 internal_keys[k] = final_content.pop(k)
  75         storage_params = internal_keys["_admin"].get("storage")
  76         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  77
  78         # 1.2. modify final_content with a serialized version
  79         final_content = copy.deepcopy(serialized)
  80         # 1.3. restore internal keys
  81         for k, v in internal_keys.items():
  82             final_content[k] = v
  83         if session["force"]:
  84             return final_content
  85
  86         # 2. check that this id is not present
  87         if "id" in edit_content:
  88             _filter = self._get_project_filter(session)
  89
  90             _filter["id"] = final_content["id"]
  91             _filter["_id.neq"] = _id
  92
  93             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  94                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  95                                                                                                final_content["id"]),
  96                                       HTTPStatus.CONFLICT)
  97
  98         return final_content
  99
 100     @staticmethod
 101     def format_on_new(content, project_id=None, make_public=False):
 102         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 103         content["_admin"]["onboardingState"] = "CREATED"
 104         content["_admin"]["operationalState"] = "DISABLED"
 105         content["_admin"]["usageState"] = "NOT_IN_USE"
 106
 107     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 108         """
 109         Deletes file system storage associated with the descriptor
 110         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 111         :param _id: server internal id
 112         :param db_content: The database content of the descriptor
 113         :param not_send_msg: To not send message (False) or store content (list) instead
 114         :return: None if ok or raises EngineException with the problem
 115         """
 116         self.fs.file_delete(_id, ignore_non_exist=True)
 117         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 118
 119     @staticmethod
 120     def get_one_by_id(db, session, topic, id):
 121         # find owned by this project
 122         _filter = BaseTopic._get_project_filter(session)
 123         _filter["id"] = id
 124         desc_list = db.get_list(topic, _filter)
 125         if len(desc_list) == 1:
 126             return desc_list[0]
 127         elif len(desc_list) > 1:
 128             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 129                               HTTPStatus.CONFLICT)
 130
 131         # not found any: try to find public
 132         _filter = BaseTopic._get_project_filter(session)
 133         _filter["id"] = id
 134         desc_list = db.get_list(topic, _filter)
 135         if not desc_list:
 136             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 137         elif len(desc_list) == 1:
 138             return desc_list[0]
 139         else:
 140             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 141                 topic[:-1], id), HTTPStatus.CONFLICT)
 142
 143     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 144         """
 145         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 146         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 147         (self.upload_content)
 148         :param rollback: list to append created items at database in case a rollback may to be done
 149         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 150         :param indata: data to be inserted
 151         :param kwargs: used to override the indata descriptor
 152         :param headers: http request headers
 153         :return: _id, None: identity of the inserted data; and None as there is not any operation
 154         """
 155
 156         # No needed to capture exceptions
 157         # Check Quota
 158         self.check_quota(session)
 159
 160         # _remove_envelop
 161         if indata:
 162             if "userDefinedData" in indata:
 163                 indata = indata['userDefinedData']
 164
 165         # Override descriptor with query string kwargs
 166         self._update_input_with_kwargs(indata, kwargs)
 167         # uncomment when this method is implemented.
 168         # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 169         # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 170
 171         content = {"_admin": {"userDefinedData": indata}}
 172         self.format_on_new(content, session["project_id"], make_public=session["public"])
 173         _id = self.db.create(self.topic, content)
 174         rollback.append({"topic": self.topic, "_id": _id})
 175         self._send_msg("created", {"_id": _id})
 176         return _id, None
 177
 178     def upload_content(self, session, _id, indata, kwargs, headers):
 179         """
 180         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 181         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 182         :param _id : the nsd,vnfd is already created, this is the id
 183         :param indata: http body request
 184         :param kwargs: user query string to override parameters. NOT USED
 185         :param headers:  http request headers
 186         :return: True if package is completely uploaded or False if partial content has been uploded
 187             Raise exception on error
 188         """
 189         # Check that _id exists and it is valid
 190         current_desc = self.show(session, _id)
 191
 192         content_range_text = headers.get("Content-Range")
 193         expected_md5 = headers.get("Content-File-MD5")
 194         compressed = None
 195         content_type = headers.get("Content-Type")
 196         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 197                 "application/zip" in content_type:
 198             compressed = "gzip"
 199         filename = headers.get("Content-Filename")
 200         if not filename:
 201             filename = "package.tar.gz" if compressed else "package"
 202         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 203         file_pkg = None
 204         error_text = ""
 205         try:
 206             if content_range_text:
 207                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 208                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 209                     raise IndexError()
 210                 start = int(content_range[1])
 211                 end = int(content_range[2]) + 1
 212                 total = int(content_range[3])
 213             else:
 214                 start = 0
 215             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 216
 217             if start:
 218                 if not self.fs.file_exists(temp_folder, 'dir'):
 219                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 220             else:
 221                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 222                 self.fs.mkdir(temp_folder)
 223
 224             storage = self.fs.get_params()
 225             storage["folder"] = _id
 226
 227             file_path = (temp_folder, filename)
 228             if self.fs.file_exists(file_path, 'file'):
 229                 file_size = self.fs.file_size(file_path)
 230             else:
 231                 file_size = 0
 232             if file_size != start:
 233                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 234                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 235             file_pkg = self.fs.file_open(file_path, 'a+b')
 236             if isinstance(indata, dict):
 237                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 238                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 239             else:
 240                 indata_len = 0
 241                 while True:
 242                     indata_text = indata.read(4096)
 243                     indata_len += len(indata_text)
 244                     if not indata_text:
 245                         break
 246                     file_pkg.write(indata_text)
 247             if content_range_text:
 248                 if indata_len != end - start:
 249                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 250                         start, end - 1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 251                 if end != total:
 252                     # TODO update to UPLOADING
 253                     return False
 254
 255             # PACKAGE UPLOADED
 256             if expected_md5:
 257                 file_pkg.seek(0, 0)
 258                 file_md5 = md5()
 259                 chunk_data = file_pkg.read(1024)
 260                 while chunk_data:
 261                     file_md5.update(chunk_data)
 262                     chunk_data = file_pkg.read(1024)
 263                 if expected_md5 != file_md5.hexdigest():
 264                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 265             file_pkg.seek(0, 0)
 266             if compressed == "gzip":
 267                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 268                 descriptor_file_name = None
 269                 for tarinfo in tar:
 270                     tarname = tarinfo.name
 271                     tarname_path = tarname.split("/")
 272                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 273                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 274                     if len(tarname_path) == 1 and not tarinfo.isdir():
 275                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 276                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 277                         storage["pkg-dir"] = tarname_path[0]
 278                         if len(tarname_path) == 2:
 279                             if descriptor_file_name:
 280                                 raise EngineException(
 281                                     "Found more than one descriptor file at package descriptor tar.gz")
 282                             descriptor_file_name = tarname
 283                 if not descriptor_file_name:
 284                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 285                 storage["descriptor"] = descriptor_file_name
 286                 storage["zipfile"] = filename
 287                 self.fs.file_extract(tar, temp_folder)
 288                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 289                     content = descriptor_file.read()
 290             else:
 291                 content = file_pkg.read()
 292                 storage["descriptor"] = descriptor_file_name = filename
 293
 294             if descriptor_file_name.endswith(".json"):
 295                 error_text = "Invalid json format "
 296                 indata = json.load(content)
 297             else:
 298                 error_text = "Invalid yaml format "
 299                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 300
 301             current_desc["_admin"]["storage"] = storage
 302             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 303             current_desc["_admin"]["operationalState"] = "ENABLED"
 304
 305             indata = self._remove_envelop(indata)
 306
 307             # Override descriptor with query string kwargs
 308             if kwargs:
 309                 self._update_input_with_kwargs(indata, kwargs)
 310
 311             deep_update_rfc7396(current_desc, indata)
 312             current_desc = self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 313             current_desc["_admin"]["modified"] = time()
 314             self.db.replace(self.topic, _id, current_desc)
 315             self.fs.dir_rename(temp_folder, _id)
 316
 317             indata["_id"] = _id
 318             self._send_msg("edited", indata)
 319
 320             # TODO if descriptor has changed because kwargs update content and remove cached zip
 321             # TODO if zip is not present creates one
 322             return True
 323
 324         except EngineException:
 325             raise
 326         except IndexError:
 327             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 328                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 329         except IOError as e:
 330             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 331         except tarfile.ReadError as e:
 332             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 333         except (ValueError, yaml.YAMLError) as e:
 334             raise EngineException(error_text + str(e))
 335         except ValidationError as e:
 336             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 337         finally:
 338             if file_pkg:
 339                 file_pkg.close()
 340
 341     def get_file(self, session, _id, path=None, accept_header=None):
 342         """
 343         Return the file content of a vnfd or nsd
 344         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 345         :param _id: Identity of the vnfd, nsd
 346         :param path: artifact path or "$DESCRIPTOR" or None
 347         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 348         :return: opened file plus Accept format or raises an exception
 349         """
 350         accept_text = accept_zip = False
 351         if accept_header:
 352             if 'text/plain' in accept_header or '*/*' in accept_header:
 353                 accept_text = True
 354             if 'application/zip' in accept_header or '*/*' in accept_header:
 355                 accept_zip = 'application/zip'
 356             elif 'application/gzip' in accept_header:
 357                 accept_zip = 'application/gzip'
 358
 359         if not accept_text and not accept_zip:
 360             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 361                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 362
 363         content = self.show(session, _id)
 364         if content["_admin"]["onboardingState"] != "ONBOARDED":
 365             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 366                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 367                                   http_code=HTTPStatus.CONFLICT)
 368         storage = content["_admin"]["storage"]
 369         if path is not None and path != "$DESCRIPTOR":  # artifacts
 370             if not storage.get('pkg-dir'):
 371                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 372             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 373                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 374                 return folder_content, "text/plain"
 375                 # TODO manage folders in http
 376             else:
 377                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \
 378                     "application/octet-stream"
 379
 380         # pkgtype   accept  ZIP  TEXT    -> result
 381         # manyfiles         yes  X       -> zip
 382         #                   no   yes     -> error
 383         # onefile           yes  no      -> zip
 384         #                   X    yes     -> text
 385         contain_many_files = False
 386         if storage.get('pkg-dir'):
 387             # check if there are more than one file in the package, ignoring checksums.txt.
 388             pkg_files = self.fs.dir_ls((storage['folder'], storage['pkg-dir']))
 389             if len(pkg_files) >= 3 or (len(pkg_files) == 2 and 'checksums.txt' not in pkg_files):
 390                 contain_many_files = True
 391         if accept_text and (not contain_many_files or path == "$DESCRIPTOR"):
 392             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 393         elif contain_many_files and not accept_zip:
 394             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 395                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 396         else:
 397             if not storage.get('zipfile'):
 398                 # TODO generate zipfile if not present
 399                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 400                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 401             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 402
 403     def _remove_yang_prefixes_from_descriptor(self, descriptor):
 404         new_descriptor = {}
 405         for k, v in descriptor.items():
 406             new_v = v
 407             if isinstance(v, dict):
 408                 new_v = self._remove_yang_prefixes_from_descriptor(v)
 409             elif isinstance(v, list):
 410                 new_v = list()
 411                 for x in v:
 412                     if isinstance(x, dict):
 413                         new_v.append(self._remove_yang_prefixes_from_descriptor(x))
 414                     else:
 415                         new_v.append(x)
 416             new_descriptor[k.split(':')[-1]] = new_v
 417         return new_descriptor
 418
 419     def pyangbind_validation(self, item, data, force=False):
 420         raise EngineException("Not possible to validate '{}' item".format(item),
 421                               http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 422
 423     def _validate_input_edit(self, indata, content, force=False):
 424         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 425         if "_id" in indata:
 426             indata.pop("_id")
 427         if "_admin" not in indata:
 428             indata["_admin"] = {}
 429
 430         if "operationalState" in indata:
 431             if indata["operationalState"] in ("ENABLED", "DISABLED"):
 432                 indata["_admin"]["operationalState"] = indata.pop("operationalState")
 433             else:
 434                 raise EngineException("State '{}' is not a valid operational state"
 435                                       .format(indata["operationalState"]),
 436                                       http_code=HTTPStatus.BAD_REQUEST)
 437
 438         # In the case of user defined data, we need to put the data in the root of the object
 439         # to preserve current expected behaviour
 440         if "userDefinedData" in indata:
 441             data = indata.pop("userDefinedData")
 442             if type(data) == dict:
 443                 indata["_admin"]["userDefinedData"] = data
 444             else:
 445                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 446                                       .format(type(data)),
 447                                       http_code=HTTPStatus.BAD_REQUEST)
 448
 449         if ("operationalState" in indata["_admin"] and
 450                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 451             raise EngineException("operationalState already {}".format(content["_admin"]["operationalState"]),
 452                                   http_code=HTTPStatus.CONFLICT)
 453
 454         return indata
 455
 456
 457 class VnfdTopic(DescriptorTopic):
 458     topic = "vnfds"
 459     topic_msg = "vnfd"
 460
 461     def __init__(self, db, fs, msg, auth):
 462         DescriptorTopic.__init__(self, db, fs, msg, auth)
 463
 464     def pyangbind_validation(self, item, data, force=False):
 465         if self._descriptor_data_is_in_old_format(data):
 466             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 467                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 468         try:
 469             myvnfd = etsi_nfv_vnfd.etsi_nfv_vnfd()
 470             pybindJSONDecoder.load_ietf_json({'etsi-nfv-vnfd:vnfd': data}, None, None, obj=myvnfd,
 471                                              path_helper=True, skip_unknown=force)
 472             out = pybindJSON.dumps(myvnfd, mode="ietf")
 473             desc_out = self._remove_envelop(yaml.safe_load(out))
 474             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 475             return utils.deep_update_dict(data, desc_out)
 476         except Exception as e:
 477             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 478                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 479
 480     @staticmethod
 481     def _descriptor_data_is_in_old_format(data):
 482         return ('vnfd-catalog' in data) or ('vnfd:vnfd-catalog' in data)
 483
 484     @staticmethod
 485     def _remove_envelop(indata=None):
 486         if not indata:
 487             return {}
 488         clean_indata = indata
 489
 490         if clean_indata.get('etsi-nfv-vnfd:vnfd'):
 491             if not isinstance(clean_indata['etsi-nfv-vnfd:vnfd'], dict):
 492                 raise EngineException("'etsi-nfv-vnfd:vnfd' must be a dict")
 493             clean_indata = clean_indata['etsi-nfv-vnfd:vnfd']
 494         elif clean_indata.get('vnfd'):
 495             if not isinstance(clean_indata['vnfd'], dict):
 496                 raise EngineException("'vnfd' must be dict")
 497             clean_indata = clean_indata['vnfd']
 498
 499         return clean_indata
 500
 501     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 502         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
 503
 504         # set type of vnfd
 505         contains_pdu = False
 506         contains_vdu = False
 507         for vdu in get_iterable(final_content.get("vdu")):
 508             if vdu.get("pdu-type"):
 509                 contains_pdu = True
 510             else:
 511                 contains_vdu = True
 512         if contains_pdu:
 513             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 514         elif contains_vdu:
 515             final_content["_admin"]["type"] = "vnfd"
 516         # if neither vud nor pdu do not fill type
 517         return final_content
 518
 519     def check_conflict_on_del(self, session, _id, db_content):
 520         """
 521         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 522         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 523         that uses this vnfd
 524         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 525         :param _id: vnfd internal id
 526         :param db_content: The database content of the _id.
 527         :return: None or raises EngineException with the conflict
 528         """
 529         if session["force"]:
 530             return
 531         descriptor = db_content
 532         descriptor_id = descriptor.get("id")
 533         if not descriptor_id:  # empty vnfd not uploaded
 534             return
 535
 536         _filter = self._get_project_filter(session)
 537
 538         # check vnfrs using this vnfd
 539         _filter["vnfd-id"] = _id
 540         if self.db.get_list("vnfrs", _filter):
 541             raise EngineException("There is at least one VNF instance using this descriptor",
 542                                   http_code=HTTPStatus.CONFLICT)
 543
 544         # check NSD referencing this VNFD
 545         del _filter["vnfd-id"]
 546         _filter["vnfd-id"] = descriptor_id
 547         if self.db.get_list("nsds", _filter):
 548             raise EngineException("There is at least one NS package referencing this descriptor",
 549                                   http_code=HTTPStatus.CONFLICT)
 550
 551     def _validate_input_new(self, indata, storage_params, force=False):
 552         indata.pop("onboardingState", None)
 553         indata.pop("operationalState", None)
 554         indata.pop("usageState", None)
 555         indata.pop("links", None)
 556
 557         indata = self.pyangbind_validation("vnfds", indata, force)
 558         # Cross references validation in the descriptor
 559
 560         self.validate_mgmt_interface_connection_point(indata)
 561
 562         for vdu in get_iterable(indata.get("vdu")):
 563             self.validate_vdu_internal_connection_points(vdu)
 564             self._validate_vdu_cloud_init_in_package(storage_params, vdu, indata)
 565         self._validate_vdu_charms_in_package(storage_params, indata)
 566
 567         self._validate_vnf_charms_in_package(storage_params, indata)
 568
 569         self.validate_external_connection_points(indata)
 570         self.validate_internal_virtual_links(indata)
 571         self.validate_monitoring_params(indata)
 572         self.validate_scaling_group_descriptor(indata)
 573
 574         return indata
 575
 576     @staticmethod
 577     def validate_mgmt_interface_connection_point(indata):
 578         if not indata.get("vdu"):
 579             return
 580         if not indata.get("mgmt-cp"):
 581             raise EngineException("'mgmt-cp' is a mandatory field and it is not defined",
 582                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 583
 584         for cp in get_iterable(indata.get("ext-cpd")):
 585             if cp["id"] == indata["mgmt-cp"]:
 586                 break
 587         else:
 588             raise EngineException("mgmt-cp='{}' must match an existing ext-cpd".format(indata["mgmt-cp"]),
 589                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 590
 591     @staticmethod
 592     def validate_vdu_internal_connection_points(vdu):
 593         int_cpds = set()
 594         for cpd in get_iterable(vdu.get("int-cpd")):
 595             cpd_id = cpd.get("id")
 596             if cpd_id and cpd_id in int_cpds:
 597                 raise EngineException("vdu[id='{}']:int-cpd[id='{}'] is already used by other int-cpd"
 598                                       .format(vdu["id"], cpd_id),
 599                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 600             int_cpds.add(cpd_id)
 601
 602     @staticmethod
 603     def validate_external_connection_points(indata):
 604         all_vdus_int_cpds = set()
 605         for vdu in get_iterable(indata.get("vdu")):
 606             for int_cpd in get_iterable(vdu.get("int-cpd")):
 607                 all_vdus_int_cpds.add((vdu.get("id"), int_cpd.get("id")))
 608
 609         ext_cpds = set()
 610         for cpd in get_iterable(indata.get("ext-cpd")):
 611             cpd_id = cpd.get("id")
 612             if cpd_id and cpd_id in ext_cpds:
 613                 raise EngineException("ext-cpd[id='{}'] is already used by other ext-cpd".format(cpd_id),
 614                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 615             ext_cpds.add(cpd_id)
 616
 617             int_cpd = cpd.get("int-cpd")
 618             if int_cpd:
 619                 if (int_cpd.get("vdu-id"), int_cpd.get("cpd")) not in all_vdus_int_cpds:
 620                     raise EngineException("ext-cpd[id='{}']:int-cpd must match an existing vdu int-cpd".format(cpd_id),
 621                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 622             # TODO: Validate k8s-cluster-net points to a valid k8s-cluster:nets ?
 623
 624     def _validate_vdu_charms_in_package(self, storage_params, indata):
 625         for df in indata["df"]:
 626             if "lcm-operations-configuration" in df and "operate-vnf-op-config" in df["lcm-operations-configuration"]:
 627                 configs = df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", [])
 628                 vdus = df.get("vdu-profile", [])
 629                 for vdu in vdus:
 630                     for config in configs:
 631                         if config["id"] == vdu["id"] and utils.find_in_list(
 632                             config.get("execution-environment-list", []),
 633                             lambda ee: "juju" in ee
 634                         ):
 635                             if not self._validate_package_folders(storage_params, 'charms'):
 636                                 raise EngineException("Charm defined in vnf[id={}] but not present in "
 637                                                       "package".format(indata["id"]))
 638
 639     def _validate_vdu_cloud_init_in_package(self, storage_params, vdu, indata):
 640         if not vdu.get("cloud-init-file"):
 641             return
 642         if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 643             raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 644                                   "package".format(indata["id"], vdu["id"]))
 645
 646     def _validate_vnf_charms_in_package(self, storage_params, indata):
 647         # Get VNF configuration through new container
 648         for deployment_flavor in indata.get('df', []):
 649             if "lcm-operations-configuration" not in deployment_flavor:
 650                 return
 651             if "operate-vnf-op-config" not in deployment_flavor["lcm-operations-configuration"]:
 652                 return
 653             for day_1_2_config in deployment_flavor["lcm-operations-configuration"]["operate-vnf-op-config"]["day1-2"]:
 654                 if day_1_2_config["id"] == indata["id"]:
 655                     if utils.find_in_list(
 656                         day_1_2_config.get("execution-environment-list", []),
 657                         lambda ee: "juju" in ee
 658                     ):
 659                         if not self._validate_package_folders(storage_params, 'charms'):
 660                             raise EngineException("Charm defined in vnf[id={}] but not present in "
 661                                                   "package".format(indata["id"]))
 662
 663     def _validate_package_folders(self, storage_params, folder, file=None):
 664         if not storage_params or not storage_params.get("pkg-dir"):
 665             return False
 666         else:
 667             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 668                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 669             else:
 670                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 671             if file:
 672                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 673             else:
 674                 if self.fs.file_exists(f, 'dir'):
 675                     if self.fs.dir_ls(f):
 676                         return True
 677             return False
 678
 679     @staticmethod
 680     def validate_internal_virtual_links(indata):
 681         all_ivld_ids = set()
 682         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 683             ivld_id = ivld.get("id")
 684             if ivld_id and ivld_id in all_ivld_ids:
 685                 raise EngineException("Duplicated VLD id in int-virtual-link-desc[id={}]".format(ivld_id),
 686                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 687             else:
 688                 all_ivld_ids.add(ivld_id)
 689
 690         for vdu in get_iterable(indata.get("vdu")):
 691             for int_cpd in get_iterable(vdu.get("int-cpd")):
 692                 int_cpd_ivld_id = int_cpd.get("int-virtual-link-desc")
 693                 if int_cpd_ivld_id and int_cpd_ivld_id not in all_ivld_ids:
 694                     raise EngineException(
 695                         "vdu[id='{}']:int-cpd[id='{}']:int-virtual-link-desc='{}' must match an existing "
 696                         "int-virtual-link-desc".format(vdu["id"], int_cpd["id"], int_cpd_ivld_id),
 697                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 698
 699         for df in get_iterable(indata.get("df")):
 700             for vlp in get_iterable(df.get("virtual-link-profile")):
 701                 vlp_ivld_id = vlp.get("id")
 702                 if vlp_ivld_id and vlp_ivld_id not in all_ivld_ids:
 703                     raise EngineException("df[id='{}']:virtual-link-profile='{}' must match an existing "
 704                                           "int-virtual-link-desc".format(df["id"], vlp_ivld_id),
 705                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 706
 707     @staticmethod
 708     def validate_monitoring_params(indata):
 709         all_monitoring_params = set()
 710         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 711             for mp in get_iterable(ivld.get("monitoring-parameters")):
 712                 mp_id = mp.get("id")
 713                 if mp_id and mp_id in all_monitoring_params:
 714                     raise EngineException("Duplicated monitoring-parameter id in "
 715                                           "int-virtual-link-desc[id='{}']:monitoring-parameters[id='{}']"
 716                                           .format(ivld["id"], mp_id),
 717                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 718                 else:
 719                     all_monitoring_params.add(mp_id)
 720
 721         for vdu in get_iterable(indata.get("vdu")):
 722             for mp in get_iterable(vdu.get("monitoring-parameter")):
 723                 mp_id = mp.get("id")
 724                 if mp_id and mp_id in all_monitoring_params:
 725                     raise EngineException("Duplicated monitoring-parameter id in "
 726                                           "vdu[id='{}']:monitoring-parameter[id='{}']"
 727                                           .format(vdu["id"], mp_id),
 728                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 729                 else:
 730                     all_monitoring_params.add(mp_id)
 731
 732         for df in get_iterable(indata.get("df")):
 733             for mp in get_iterable(df.get("monitoring-parameter")):
 734                 mp_id = mp.get("id")
 735                 if mp_id and mp_id in all_monitoring_params:
 736                     raise EngineException("Duplicated monitoring-parameter id in "
 737                                           "df[id='{}']:monitoring-parameter[id='{}']"
 738                                           .format(df["id"], mp_id),
 739                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 740                 else:
 741                     all_monitoring_params.add(mp_id)
 742
 743     @staticmethod
 744     def validate_scaling_group_descriptor(indata):
 745         all_monitoring_params = set()
 746         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 747             for mp in get_iterable(ivld.get("monitoring-parameters")):
 748                 all_monitoring_params.add(mp.get("id"))
 749
 750         for vdu in get_iterable(indata.get("vdu")):
 751             for mp in get_iterable(vdu.get("monitoring-parameter")):
 752                 all_monitoring_params.add(mp.get("id"))
 753
 754         for df in get_iterable(indata.get("df")):
 755             for mp in get_iterable(df.get("monitoring-parameter")):
 756                 all_monitoring_params.add(mp.get("id"))
 757
 758         for df in get_iterable(indata.get("df")):
 759             for sa in get_iterable(df.get("scaling-aspect")):
 760                 for sp in get_iterable(sa.get("scaling-policy")):
 761                     for sc in get_iterable(sp.get("scaling-criteria")):
 762                         sc_monitoring_param = sc.get("vnf-monitoring-param-ref")
 763                         if sc_monitoring_param and sc_monitoring_param not in all_monitoring_params:
 764                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-policy"
 765                                                   "[name='{}']:scaling-criteria[name='{}']: "
 766                                                   "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 767                                                   .format(df["id"], sa["id"], sp["name"], sc["name"],
 768                                                           sc_monitoring_param),
 769                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 770
 771                 for sca in get_iterable(sa.get("scaling-config-action")):
 772                     if "lcm-operations-configuration" not in df \
 773                         or "operate-vnf-op-config" not in df["lcm-operations-configuration"] \
 774                         or not utils.find_in_list(
 775                             df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", []),
 776                             lambda config: config["id"] == indata["id"]):
 777                         raise EngineException("'day1-2 configuration' not defined in the descriptor but it is "
 778                                               "referenced by df[id='{}']:scaling-aspect[id='{}']:scaling-config-action"
 779                                               .format(df["id"], sa["id"]),
 780                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 781                     for configuration in get_iterable(
 782                         df["lcm-operations-configuration"]["operate-vnf-op-config"].get("day1-2", [])
 783                     ):
 784                         for primitive in get_iterable(configuration.get("config-primitive")):
 785                             if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 786                                 break
 787                         else:
 788                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-config-action:vnf-"
 789                                                   "config-primitive-name-ref='{}' does not match any "
 790                                                   "day1-2 configuration:config-primitive:name"
 791                                                   .format(df["id"], sa["id"], sca["vnf-config-primitive-name-ref"]),
 792                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 793
 794     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 795         """
 796         Deletes associate file system storage (via super)
 797         Deletes associated vnfpkgops from database.
 798         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 799         :param _id: server internal id
 800         :param db_content: The database content of the descriptor
 801         :return: None
 802         :raises: FsException in case of error while deleting associated storage
 803         """
 804         super().delete_extra(session, _id, db_content, not_send_msg)
 805         self.db.del_list("vnfpkgops", {"vnfPkgId": _id})
 806
 807     def sol005_projection(self, data):
 808         data["onboardingState"] = data["_admin"]["onboardingState"]
 809         data["operationalState"] = data["_admin"]["operationalState"]
 810         data["usageState"] = data["_admin"]["usageState"]
 811
 812         links = {}
 813         links["self"] = {"href": "/vnfpkgm/v1/vnf_packages/{}".format(data["_id"])}
 814         links["vnfd"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(data["_id"])}
 815         links["packageContent"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/package_content".format(data["_id"])}
 816         data["_links"] = links
 817
 818         return super().sol005_projection(data)
 819
 820
 821 class NsdTopic(DescriptorTopic):
 822     topic = "nsds"
 823     topic_msg = "nsd"
 824
 825     def __init__(self, db, fs, msg, auth):
 826         DescriptorTopic.__init__(self, db, fs, msg, auth)
 827
 828     def pyangbind_validation(self, item, data, force=False):
 829         if self._descriptor_data_is_in_old_format(data):
 830             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 831                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 832         try:
 833             nsd_vnf_profiles = data.get('df', [{}])[0].get('vnf-profile', [])
 834             mynsd = etsi_nfv_nsd.etsi_nfv_nsd()
 835             pybindJSONDecoder.load_ietf_json({'nsd': {'nsd': [data]}}, None, None, obj=mynsd,
 836                                              path_helper=True, skip_unknown=force)
 837             out = pybindJSON.dumps(mynsd, mode="ietf")
 838             desc_out = self._remove_envelop(yaml.safe_load(out))
 839             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 840             if nsd_vnf_profiles:
 841                 desc_out['df'][0]['vnf-profile'] = nsd_vnf_profiles
 842             return desc_out
 843         except Exception as e:
 844             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 845                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 846
 847     @staticmethod
 848     def _descriptor_data_is_in_old_format(data):
 849         return ('nsd-catalog' in data) or ('nsd:nsd-catalog' in data)
 850
 851     @staticmethod
 852     def _remove_envelop(indata=None):
 853         if not indata:
 854             return {}
 855         clean_indata = indata
 856
 857         if clean_indata.get('nsd'):
 858             clean_indata = clean_indata['nsd']
 859         elif clean_indata.get('etsi-nfv-nsd:nsd'):
 860             clean_indata = clean_indata['etsi-nfv-nsd:nsd']
 861         if clean_indata.get('nsd'):
 862             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 863                 raise EngineException("'nsd' must be a list of only one element")
 864             clean_indata = clean_indata['nsd'][0]
 865         return clean_indata
 866
 867     def _validate_input_new(self, indata, storage_params, force=False):
 868         indata.pop("nsdOnboardingState", None)
 869         indata.pop("nsdOperationalState", None)
 870         indata.pop("nsdUsageState", None)
 871
 872         indata.pop("links", None)
 873
 874         indata = self.pyangbind_validation("nsds", indata, force)
 875         # Cross references validation in the descriptor
 876         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 877         for vld in get_iterable(indata.get("virtual-link-desc")):
 878             self.validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata)
 879
 880         self.validate_vnf_profiles_vnfd_id(indata)
 881
 882         return indata
 883
 884     @staticmethod
 885     def validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata):
 886         if not vld.get("mgmt-network"):
 887             return
 888         vld_id = vld.get("id")
 889         for df in get_iterable(indata.get("df")):
 890             for vlp in get_iterable(df.get("virtual-link-profile")):
 891                 if vld_id and vld_id == vlp.get("virtual-link-desc-id"):
 892                     if vlp.get("virtual-link-protocol-data"):
 893                         raise EngineException("Error at df[id='{}']:virtual-link-profile[id='{}']:virtual-link-"
 894                                               "protocol-data You cannot set a virtual-link-protocol-data "
 895                                               "when mgmt-network is True"
 896                                               .format(df["id"], vlp["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 897
 898     @staticmethod
 899     def validate_vnf_profiles_vnfd_id(indata):
 900         all_vnfd_ids = set(get_iterable(indata.get("vnfd-id")))
 901         for df in get_iterable(indata.get("df")):
 902             for vnf_profile in get_iterable(df.get("vnf-profile")):
 903                 vnfd_id = vnf_profile.get("vnfd-id")
 904                 if vnfd_id and vnfd_id not in all_vnfd_ids:
 905                     raise EngineException("Error at df[id='{}']:vnf_profile[id='{}']:vnfd-id='{}' "
 906                                           "does not match any vnfd-id".format(df["id"], vnf_profile["id"], vnfd_id),
 907                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 908
 909     def _validate_input_edit(self, indata, content, force=False):
 910         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 911         """
 912         indata looks as follows:
 913             - In the new case (conformant)
 914                 {'nsdOperationalState': 'DISABLED', 'userDefinedData': {'id': 'string23',
 915                 '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}}
 916             - In the old case (backwards-compatible)
 917                 {'id': 'string23', '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}
 918         """
 919         if "_admin" not in indata:
 920             indata["_admin"] = {}
 921
 922         if "nsdOperationalState" in indata:
 923             if indata["nsdOperationalState"] in ("ENABLED", "DISABLED"):
 924                 indata["_admin"]["operationalState"] = indata.pop("nsdOperationalState")
 925             else:
 926                 raise EngineException("State '{}' is not a valid operational state"
 927                                       .format(indata["nsdOperationalState"]),
 928                                       http_code=HTTPStatus.BAD_REQUEST)
 929
 930         # In the case of user defined data, we need to put the data in the root of the object
 931         # to preserve current expected behaviour
 932         if "userDefinedData" in indata:
 933             data = indata.pop("userDefinedData")
 934             if type(data) == dict:
 935                 indata["_admin"]["userDefinedData"] = data
 936             else:
 937                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 938                                       .format(type(data)),
 939                                       http_code=HTTPStatus.BAD_REQUEST)
 940         if ("operationalState" in indata["_admin"] and
 941                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 942             raise EngineException("nsdOperationalState already {}".format(content["_admin"]["operationalState"]),
 943                                   http_code=HTTPStatus.CONFLICT)
 944         return indata
 945
 946     def _check_descriptor_dependencies(self, session, descriptor):
 947         """
 948         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 949         connection points are ok
 950         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 951         :param descriptor: descriptor to be inserted or edit
 952         :return: None or raises exception
 953         """
 954         if session["force"]:
 955             return
 956         vnfds_index = self._get_descriptor_constituent_vnfds_index(session, descriptor)
 957
 958         # Cross references validation in the descriptor and vnfd connection point validation
 959         for df in get_iterable(descriptor.get("df")):
 960             self.validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index)
 961
 962     def _get_descriptor_constituent_vnfds_index(self, session, descriptor):
 963         vnfds_index = {}
 964         if descriptor.get("vnfd-id") and not session["force"]:
 965             for vnfd_id in get_iterable(descriptor.get("vnfd-id")):
 966                 query_filter = self._get_project_filter(session)
 967                 query_filter["id"] = vnfd_id
 968                 vnf_list = self.db.get_list("vnfds", query_filter)
 969                 if not vnf_list:
 970                     raise EngineException("Descriptor error at 'vnfd-id'='{}' references a non "
 971                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 972                 vnfds_index[vnfd_id] = vnf_list[0]
 973         return vnfds_index
 974
 975     @staticmethod
 976     def validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index):
 977         for vnf_profile in get_iterable(df.get("vnf-profile")):
 978             vnfd = vnfds_index.get(vnf_profile["vnfd-id"])
 979             all_vnfd_ext_cpds = set()
 980             for ext_cpd in get_iterable(vnfd.get("ext-cpd")):
 981                 if ext_cpd.get('id'):
 982                     all_vnfd_ext_cpds.add(ext_cpd.get('id'))
 983
 984             for virtual_link in get_iterable(vnf_profile.get("virtual-link-connectivity")):
 985                 for vl_cpd in get_iterable(virtual_link.get("constituent-cpd-id")):
 986                     vl_cpd_id = vl_cpd.get('constituent-cpd-id')
 987                     if vl_cpd_id and vl_cpd_id not in all_vnfd_ext_cpds:
 988                         raise EngineException("Error at df[id='{}']:vnf-profile[id='{}']:virtual-link-connectivity"
 989                                               "[virtual-link-profile-id='{}']:constituent-cpd-id='{}' references a "
 990                                               "non existing ext-cpd:id inside vnfd '{}'"
 991                                               .format(df["id"], vnf_profile["id"],
 992                                                       virtual_link["virtual-link-profile-id"], vl_cpd_id, vnfd["id"]),
 993                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 994
 995     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 996         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
 997
 998         self._check_descriptor_dependencies(session, final_content)
 999
1000         return final_content
1001
1002     def check_conflict_on_del(self, session, _id, db_content):
1003         """
1004         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
1005         that NSD can be public and be used by other projects.
1006         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1007         :param _id: nsd internal id
1008         :param db_content: The database content of the _id
1009         :return: None or raises EngineException with the conflict
1010         """
1011         if session["force"]:
1012             return
1013         descriptor = db_content
1014         descriptor_id = descriptor.get("id")
1015         if not descriptor_id:  # empty nsd not uploaded
1016             return
1017
1018         # check NSD used by NS
1019         _filter = self._get_project_filter(session)
1020         _filter["nsd-id"] = _id
1021         if self.db.get_list("nsrs", _filter):
1022             raise EngineException("There is at least one NS instance using this descriptor",
1023                                   http_code=HTTPStatus.CONFLICT)
1024
1025         # check NSD referenced by NST
1026         del _filter["nsd-id"]
1027         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
1028         if self.db.get_list("nsts", _filter):
1029             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
1030                                   http_code=HTTPStatus.CONFLICT)
1031
1032     def sol005_projection(self, data):
1033         data["nsdOnboardingState"] = data["_admin"]["onboardingState"]
1034         data["nsdOperationalState"] = data["_admin"]["operationalState"]
1035         data["nsdUsageState"] = data["_admin"]["usageState"]
1036
1037         links = {}
1038         links["self"] = {"href": "/nsd/v1/ns_descriptors/{}".format(data["_id"])}
1039         links["nsd_content"] = {"href": "/nsd/v1/ns_descriptors/{}/nsd_content".format(data["_id"])}
1040         data["_links"] = links
1041
1042         return super().sol005_projection(data)
1043
1044
1045 class NstTopic(DescriptorTopic):
1046     topic = "nsts"
1047     topic_msg = "nst"
1048     quota_name = "slice_templates"
1049
1050     def __init__(self, db, fs, msg, auth):
1051         DescriptorTopic.__init__(self, db, fs, msg, auth)
1052
1053     def pyangbind_validation(self, item, data, force=False):
1054         try:
1055             mynst = nst_im()
1056             pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
1057                                              path_helper=True, skip_unknown=force)
1058             out = pybindJSON.dumps(mynst, mode="ietf")
1059             desc_out = self._remove_envelop(yaml.safe_load(out))
1060             return desc_out
1061         except Exception as e:
1062             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
1063                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
1064
1065     @staticmethod
1066     def _remove_envelop(indata=None):
1067         if not indata:
1068             return {}
1069         clean_indata = indata
1070
1071         if clean_indata.get('nst'):
1072             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
1073                 raise EngineException("'nst' must be a list only one element")
1074             clean_indata = clean_indata['nst'][0]
1075         elif clean_indata.get('nst:nst'):
1076             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
1077                 raise EngineException("'nst:nst' must be a list only one element")
1078             clean_indata = clean_indata['nst:nst'][0]
1079         return clean_indata
1080
1081     def _validate_input_new(self, indata, storage_params, force=False):
1082         indata.pop("onboardingState", None)
1083         indata.pop("operationalState", None)
1084         indata.pop("usageState", None)
1085         indata = self.pyangbind_validation("nsts", indata, force)
1086         return indata.copy()
1087
1088     def _check_descriptor_dependencies(self, session, descriptor):
1089         """
1090         Check that the dependent descriptors exist on a new descriptor or edition
1091         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1092         :param descriptor: descriptor to be inserted or edit
1093         :return: None or raises exception
1094         """
1095         if not descriptor.get("netslice-subnet"):
1096             return
1097         for nsd in descriptor["netslice-subnet"]:
1098             nsd_id = nsd["nsd-ref"]
1099             filter_q = self._get_project_filter(session)
1100             filter_q["id"] = nsd_id
1101             if not self.db.get_list("nsds", filter_q):
1102                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
1103                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
1104
1105     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
1106         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
1107
1108         self._check_descriptor_dependencies(session, final_content)
1109         return final_content
1110
1111     def check_conflict_on_del(self, session, _id, db_content):
1112         """
1113         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
1114         that NST can be public and be used by other projects.
1115         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1116         :param _id: nst internal id
1117         :param db_content: The database content of the _id.
1118         :return: None or raises EngineException with the conflict
1119         """
1120         # TODO: Check this method
1121         if session["force"]:
1122             return
1123         # Get Network Slice Template from Database
1124         _filter = self._get_project_filter(session)
1125         _filter["_admin.nst-id"] = _id
1126         if self.db.get_list("nsis", _filter):
1127             raise EngineException("there is at least one Netslice Instance using this descriptor",
1128                                   http_code=HTTPStatus.CONFLICT)
1129
1130     def sol005_projection(self, data):
1131         data["onboardingState"] = data["_admin"]["onboardingState"]
1132         data["operationalState"] = data["_admin"]["operationalState"]
1133         data["usageState"] = data["_admin"]["usageState"]
1134
1135         links = {}
1136         links["self"] = {"href": "/nst/v1/netslice_templates/{}".format(data["_id"])}
1137         links["nst"] = {"href": "/nst/v1/netslice_templates/{}/nst".format(data["_id"])}
1138         data["_links"] = links
1139
1140         return super().sol005_projection(data)
1141
1142
1143 class PduTopic(BaseTopic):
1144     topic = "pdus"
1145     topic_msg = "pdu"
1146     quota_name = "pduds"
1147     schema_new = pdu_new_schema
1148     schema_edit = pdu_edit_schema
1149
1150     def __init__(self, db, fs, msg, auth):
1151         BaseTopic.__init__(self, db, fs, msg, auth)
1152
1153     @staticmethod
1154     def format_on_new(content, project_id=None, make_public=False):
1155         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
1156         content["_admin"]["onboardingState"] = "CREATED"
1157         content["_admin"]["operationalState"] = "ENABLED"
1158         content["_admin"]["usageState"] = "NOT_IN_USE"
1159
1160     def check_conflict_on_del(self, session, _id, db_content):
1161         """
1162         Check that there is not any vnfr that uses this PDU
1163         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1164         :param _id: pdu internal id
1165         :param db_content: The database content of the _id.
1166         :return: None or raises EngineException with the conflict
1167         """
1168         if session["force"]:
1169             return
1170
1171         _filter = self._get_project_filter(session)
1172         _filter["vdur.pdu-id"] = _id
1173         if self.db.get_list("vnfrs", _filter):
1174             raise EngineException("There is at least one VNF instance using this PDU", http_code=HTTPStatus.CONFLICT)
1175
1176
1177 class VnfPkgOpTopic(BaseTopic):
1178     topic = "vnfpkgops"
1179     topic_msg = "vnfd"
1180     schema_new = vnfpkgop_new_schema
1181     schema_edit = None
1182
1183     def __init__(self, db, fs, msg, auth):
1184         BaseTopic.__init__(self, db, fs, msg, auth)
1185
1186     def edit(self, session, _id, indata=None, kwargs=None, content=None):
1187         raise EngineException("Method 'edit' not allowed for topic '{}'".format(self.topic),
1188                               HTTPStatus.METHOD_NOT_ALLOWED)
1189
1190     def delete(self, session, _id, dry_run=False):
1191         raise EngineException("Method 'delete' not allowed for topic '{}'".format(self.topic),
1192                               HTTPStatus.METHOD_NOT_ALLOWED)
1193
1194     def delete_list(self, session, filter_q=None):
1195         raise EngineException("Method 'delete_list' not allowed for topic '{}'".format(self.topic),
1196                               HTTPStatus.METHOD_NOT_ALLOWED)
1197
1198     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
1199         """
1200         Creates a new entry into database.
1201         :param rollback: list to append created items at database in case a rollback may to be done
1202         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1203         :param indata: data to be inserted
1204         :param kwargs: used to override the indata descriptor
1205         :param headers: http request headers
1206         :return: _id, op_id:
1207             _id: identity of the inserted data.
1208              op_id: None
1209         """
1210         self._update_input_with_kwargs(indata, kwargs)
1211         validate_input(indata, self.schema_new)
1212         vnfpkg_id = indata["vnfPkgId"]
1213         filter_q = BaseTopic._get_project_filter(session)
1214         filter_q["_id"] = vnfpkg_id
1215         vnfd = self.db.get_one("vnfds", filter_q)
1216         operation = indata["lcmOperationType"]
1217         kdu_name = indata["kdu_name"]
1218         for kdu in vnfd.get("kdu", []):
1219             if kdu["name"] == kdu_name:
1220                 helm_chart = kdu.get("helm-chart")
1221                 juju_bundle = kdu.get("juju-bundle")
1222                 break
1223         else:
1224             raise EngineException("Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id, kdu_name))
1225         if helm_chart:
1226             indata["helm-chart"] = helm_chart
1227             match = fullmatch(r"([^/]*)/([^/]*)", helm_chart)
1228             repo_name = match.group(1) if match else None
1229         elif juju_bundle:
1230             indata["juju-bundle"] = juju_bundle
1231             match = fullmatch(r"([^/]*)/([^/]*)", juju_bundle)
1232             repo_name = match.group(1) if match else None
1233         else:
1234             raise EngineException("Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']"
1235                                   .format(vnfpkg_id, kdu_name))
1236         if repo_name:
1237             del filter_q["_id"]
1238             filter_q["name"] = repo_name
1239             repo = self.db.get_one("k8srepos", filter_q)
1240             k8srepo_id = repo.get("_id")
1241             k8srepo_url = repo.get("url")
1242         else:
1243             k8srepo_id = None
1244             k8srepo_url = None
1245         indata["k8srepoId"] = k8srepo_id
1246         indata["k8srepo_url"] = k8srepo_url
1247         vnfpkgop_id = str(uuid4())
1248         vnfpkgop_desc = {
1249             "_id": vnfpkgop_id,
1250             "operationState": "PROCESSING",
1251             "vnfPkgId": vnfpkg_id,
1252             "lcmOperationType": operation,
1253             "isAutomaticInvocation": False,
1254             "isCancelPending": False,
1255             "operationParams": indata,
1256             "links": {
1257                 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id,
1258                 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id,
1259             }
1260         }
1261         self.format_on_new(vnfpkgop_desc, session["project_id"], make_public=session["public"])
1262         ctime = vnfpkgop_desc["_admin"]["created"]
1263         vnfpkgop_desc["statusEnteredTime"] = ctime
1264         vnfpkgop_desc["startTime"] = ctime
1265         self.db.create(self.topic, vnfpkgop_desc)
1266         rollback.append({"topic": self.topic, "_id": vnfpkgop_id})
1267         self.msg.write(self.topic_msg, operation, vnfpkgop_desc)
1268         return vnfpkgop_id, None