osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 import importlib
  20 import copy
  21 # import logging
  22 from hashlib import md5
  23 from osm_common.dbbase import DbException, deep_update_rfc7396
  24 from http import HTTPStatus
  25 from time import time
  26 from uuid import uuid4
  27 from re import fullmatch
  28 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema, \
  29     validate_input, vnfpkgop_new_schema
  30 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  31 etsi_nfv_vnfd = importlib.import_module("osm_im.etsi-nfv-vnfd")
  32 etsi_nfv_nsd = importlib.import_module("osm_im.etsi-nfv-nsd")
  33 from osm_im.nst import nst as nst_im
  34 from pyangbind.lib.serialise import pybindJSONDecoder
  35 import pyangbind.lib.pybindJSON as pybindJSON
  36 from osm_nbi.utils import deep_update_dict
  37
  38 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  39
  40
  41 class DescriptorTopic(BaseTopic):
  42
  43     def __init__(self, db, fs, msg, auth):
  44         BaseTopic.__init__(self, db, fs, msg, auth)
  45
  46     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  47         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
  48
  49         def _check_unique_id_name(descriptor, position=""):
  50             for desc_key, desc_item in descriptor.items():
  51                 if isinstance(desc_item, list) and desc_item:
  52                     used_ids = []
  53                     desc_item_id = None
  54                     for index, list_item in enumerate(desc_item):
  55                         if isinstance(list_item, dict):
  56                             _check_unique_id_name(list_item, "{}.{}[{}]"
  57                                                   .format(position, desc_key, index))
  58                             # Base case
  59                             if index == 0 and (list_item.get("id") or list_item.get("name")):
  60                                 desc_item_id = "id" if list_item.get("id") else "name"
  61                             if desc_item_id and list_item.get(desc_item_id):
  62                                 if list_item[desc_item_id] in used_ids:
  63                                     position = "{}.{}[{}]".format(position, desc_key, index)
  64                                     raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
  65                                                           .format(desc_item_id, list_item[desc_item_id],
  66                                                                   position), HTTPStatus.UNPROCESSABLE_ENTITY)
  67                                 used_ids.append(list_item[desc_item_id])
  68
  69         _check_unique_id_name(final_content)
  70         # 1. validate again with pyangbind
  71         # 1.1. remove internal keys
  72         internal_keys = {}
  73         for k in ("_id", "_admin"):
  74             if k in final_content:
  75                 internal_keys[k] = final_content.pop(k)
  76         storage_params = internal_keys["_admin"].get("storage")
  77         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  78
  79         # 1.2. modify final_content with a serialized version
  80         final_content = copy.deepcopy(serialized)
  81         # 1.3. restore internal keys
  82         for k, v in internal_keys.items():
  83             final_content[k] = v
  84         if session["force"]:
  85             return final_content
  86
  87         # 2. check that this id is not present
  88         if "id" in edit_content:
  89             _filter = self._get_project_filter(session)
  90
  91             _filter["id"] = final_content["id"]
  92             _filter["_id.neq"] = _id
  93
  94             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  95                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  96                                                                                                final_content["id"]),
  97                                       HTTPStatus.CONFLICT)
  98
  99         return final_content
 100
 101     @staticmethod
 102     def format_on_new(content, project_id=None, make_public=False):
 103         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 104         content["_admin"]["onboardingState"] = "CREATED"
 105         content["_admin"]["operationalState"] = "DISABLED"
 106         content["_admin"]["usageState"] = "NOT_IN_USE"
 107
 108     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 109         """
 110         Deletes file system storage associated with the descriptor
 111         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 112         :param _id: server internal id
 113         :param db_content: The database content of the descriptor
 114         :param not_send_msg: To not send message (False) or store content (list) instead
 115         :return: None if ok or raises EngineException with the problem
 116         """
 117         self.fs.file_delete(_id, ignore_non_exist=True)
 118         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
 119
 120     @staticmethod
 121     def get_one_by_id(db, session, topic, id):
 122         # find owned by this project
 123         _filter = BaseTopic._get_project_filter(session)
 124         _filter["id"] = id
 125         desc_list = db.get_list(topic, _filter)
 126         if len(desc_list) == 1:
 127             return desc_list[0]
 128         elif len(desc_list) > 1:
 129             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
 130                               HTTPStatus.CONFLICT)
 131
 132         # not found any: try to find public
 133         _filter = BaseTopic._get_project_filter(session)
 134         _filter["id"] = id
 135         desc_list = db.get_list(topic, _filter)
 136         if not desc_list:
 137             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 138         elif len(desc_list) == 1:
 139             return desc_list[0]
 140         else:
 141             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 142                 topic[:-1], id), HTTPStatus.CONFLICT)
 143
 144     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 145         """
 146         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 147         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 148         (self.upload_content)
 149         :param rollback: list to append created items at database in case a rollback may to be done
 150         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 151         :param indata: data to be inserted
 152         :param kwargs: used to override the indata descriptor
 153         :param headers: http request headers
 154         :return: _id, None: identity of the inserted data; and None as there is not any operation
 155         """
 156
 157         # No needed to capture exceptions
 158         # Check Quota
 159         self.check_quota(session)
 160
 161         # _remove_envelop
 162         if indata:
 163             if "userDefinedData" in indata:
 164                 indata = indata['userDefinedData']
 165
 166         # Override descriptor with query string kwargs
 167         self._update_input_with_kwargs(indata, kwargs)
 168         # uncomment when this method is implemented.
 169         # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 170         # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 171
 172         content = {"_admin": {"userDefinedData": indata}}
 173         self.format_on_new(content, session["project_id"], make_public=session["public"])
 174         _id = self.db.create(self.topic, content)
 175         rollback.append({"topic": self.topic, "_id": _id})
 176         self._send_msg("created", {"_id": _id})
 177         return _id, None
 178
 179     def upload_content(self, session, _id, indata, kwargs, headers):
 180         """
 181         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 182         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 183         :param _id : the nsd,vnfd is already created, this is the id
 184         :param indata: http body request
 185         :param kwargs: user query string to override parameters. NOT USED
 186         :param headers:  http request headers
 187         :return: True if package is completely uploaded or False if partial content has been uploded
 188             Raise exception on error
 189         """
 190         # Check that _id exists and it is valid
 191         current_desc = self.show(session, _id)
 192
 193         content_range_text = headers.get("Content-Range")
 194         expected_md5 = headers.get("Content-File-MD5")
 195         compressed = None
 196         content_type = headers.get("Content-Type")
 197         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 198                 "application/zip" in content_type:
 199             compressed = "gzip"
 200         filename = headers.get("Content-Filename")
 201         if not filename:
 202             filename = "package.tar.gz" if compressed else "package"
 203         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 204         file_pkg = None
 205         error_text = ""
 206         try:
 207             if content_range_text:
 208                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 209                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 210                     raise IndexError()
 211                 start = int(content_range[1])
 212                 end = int(content_range[2]) + 1
 213                 total = int(content_range[3])
 214             else:
 215                 start = 0
 216             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 217
 218             if start:
 219                 if not self.fs.file_exists(temp_folder, 'dir'):
 220                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 221             else:
 222                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 223                 self.fs.mkdir(temp_folder)
 224
 225             storage = self.fs.get_params()
 226             storage["folder"] = _id
 227
 228             file_path = (temp_folder, filename)
 229             if self.fs.file_exists(file_path, 'file'):
 230                 file_size = self.fs.file_size(file_path)
 231             else:
 232                 file_size = 0
 233             if file_size != start:
 234                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 235                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 236             file_pkg = self.fs.file_open(file_path, 'a+b')
 237             if isinstance(indata, dict):
 238                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 239                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 240             else:
 241                 indata_len = 0
 242                 while True:
 243                     indata_text = indata.read(4096)
 244                     indata_len += len(indata_text)
 245                     if not indata_text:
 246                         break
 247                     file_pkg.write(indata_text)
 248             if content_range_text:
 249                 if indata_len != end - start:
 250                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 251                         start, end - 1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 252                 if end != total:
 253                     # TODO update to UPLOADING
 254                     return False
 255
 256             # PACKAGE UPLOADED
 257             if expected_md5:
 258                 file_pkg.seek(0, 0)
 259                 file_md5 = md5()
 260                 chunk_data = file_pkg.read(1024)
 261                 while chunk_data:
 262                     file_md5.update(chunk_data)
 263                     chunk_data = file_pkg.read(1024)
 264                 if expected_md5 != file_md5.hexdigest():
 265                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 266             file_pkg.seek(0, 0)
 267             if compressed == "gzip":
 268                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 269                 descriptor_file_name = None
 270                 for tarinfo in tar:
 271                     tarname = tarinfo.name
 272                     tarname_path = tarname.split("/")
 273                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 274                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 275                     if len(tarname_path) == 1 and not tarinfo.isdir():
 276                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 277                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 278                         storage["pkg-dir"] = tarname_path[0]
 279                         if len(tarname_path) == 2:
 280                             if descriptor_file_name:
 281                                 raise EngineException(
 282                                     "Found more than one descriptor file at package descriptor tar.gz")
 283                             descriptor_file_name = tarname
 284                 if not descriptor_file_name:
 285                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 286                 storage["descriptor"] = descriptor_file_name
 287                 storage["zipfile"] = filename
 288                 self.fs.file_extract(tar, temp_folder)
 289                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 290                     content = descriptor_file.read()
 291             else:
 292                 content = file_pkg.read()
 293                 storage["descriptor"] = descriptor_file_name = filename
 294
 295             if descriptor_file_name.endswith(".json"):
 296                 error_text = "Invalid json format "
 297                 indata = json.load(content)
 298             else:
 299                 error_text = "Invalid yaml format "
 300                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 301
 302             current_desc["_admin"]["storage"] = storage
 303             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 304             current_desc["_admin"]["operationalState"] = "ENABLED"
 305
 306             indata = self._remove_envelop(indata)
 307
 308             # Override descriptor with query string kwargs
 309             if kwargs:
 310                 self._update_input_with_kwargs(indata, kwargs)
 311
 312             deep_update_rfc7396(current_desc, indata)
 313             current_desc = self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 314             current_desc["_admin"]["modified"] = time()
 315             self.db.replace(self.topic, _id, current_desc)
 316             self.fs.dir_rename(temp_folder, _id)
 317
 318             indata["_id"] = _id
 319             self._send_msg("edited", indata)
 320
 321             # TODO if descriptor has changed because kwargs update content and remove cached zip
 322             # TODO if zip is not present creates one
 323             return True
 324
 325         except EngineException:
 326             raise
 327         except IndexError:
 328             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 329                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 330         except IOError as e:
 331             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 332         except tarfile.ReadError as e:
 333             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 334         except (ValueError, yaml.YAMLError) as e:
 335             raise EngineException(error_text + str(e))
 336         except ValidationError as e:
 337             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 338         finally:
 339             if file_pkg:
 340                 file_pkg.close()
 341
 342     def get_file(self, session, _id, path=None, accept_header=None):
 343         """
 344         Return the file content of a vnfd or nsd
 345         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 346         :param _id: Identity of the vnfd, nsd
 347         :param path: artifact path or "$DESCRIPTOR" or None
 348         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 349         :return: opened file plus Accept format or raises an exception
 350         """
 351         accept_text = accept_zip = False
 352         if accept_header:
 353             if 'text/plain' in accept_header or '*/*' in accept_header:
 354                 accept_text = True
 355             if 'application/zip' in accept_header or '*/*' in accept_header:
 356                 accept_zip = 'application/zip'
 357             elif 'application/gzip' in accept_header:
 358                 accept_zip = 'application/gzip'
 359
 360         if not accept_text and not accept_zip:
 361             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 362                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 363
 364         content = self.show(session, _id)
 365         if content["_admin"]["onboardingState"] != "ONBOARDED":
 366             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 367                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 368                                   http_code=HTTPStatus.CONFLICT)
 369         storage = content["_admin"]["storage"]
 370         if path is not None and path != "$DESCRIPTOR":  # artifacts
 371             if not storage.get('pkg-dir'):
 372                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 373             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 374                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 375                 return folder_content, "text/plain"
 376                 # TODO manage folders in http
 377             else:
 378                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \
 379                     "application/octet-stream"
 380
 381         # pkgtype   accept  ZIP  TEXT    -> result
 382         # manyfiles         yes  X       -> zip
 383         #                   no   yes     -> error
 384         # onefile           yes  no      -> zip
 385         #                   X    yes     -> text
 386         contain_many_files = False
 387         if storage.get('pkg-dir'):
 388             # check if there are more than one file in the package, ignoring checksums.txt.
 389             pkg_files = self.fs.dir_ls((storage['folder'], storage['pkg-dir']))
 390             if len(pkg_files) >= 3 or (len(pkg_files) == 2 and 'checksums.txt' not in pkg_files):
 391                 contain_many_files = True
 392         if accept_text and (not contain_many_files or path == "$DESCRIPTOR"):
 393             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 394         elif contain_many_files and not accept_zip:
 395             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 396                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 397         else:
 398             if not storage.get('zipfile'):
 399                 # TODO generate zipfile if not present
 400                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 401                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 402             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 403
 404     def _remove_yang_prefixes_from_descriptor(self, descriptor):
 405         new_descriptor = {}
 406         for k, v in descriptor.items():
 407             new_v = v
 408             if isinstance(v, dict):
 409                 new_v = self._remove_yang_prefixes_from_descriptor(v)
 410             elif isinstance(v, list):
 411                 new_v = list()
 412                 for x in v:
 413                     if isinstance(x, dict):
 414                         new_v.append(self._remove_yang_prefixes_from_descriptor(x))
 415                     else:
 416                         new_v.append(x)
 417             new_descriptor[k.split(':')[-1]] = new_v
 418         return new_descriptor
 419
 420     def pyangbind_validation(self, item, data, force=False):
 421         raise EngineException("Not possible to validate '{}' item".format(item),
 422                               http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 423
 424     def _validate_input_edit(self, indata, content, force=False):
 425         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 426         if "_id" in indata:
 427             indata.pop("_id")
 428         if "_admin" not in indata:
 429             indata["_admin"] = {}
 430
 431         if "operationalState" in indata:
 432             if indata["operationalState"] in ("ENABLED", "DISABLED"):
 433                 indata["_admin"]["operationalState"] = indata.pop("operationalState")
 434             else:
 435                 raise EngineException("State '{}' is not a valid operational state"
 436                                       .format(indata["operationalState"]),
 437                                       http_code=HTTPStatus.BAD_REQUEST)
 438
 439         # In the case of user defined data, we need to put the data in the root of the object
 440         # to preserve current expected behaviour
 441         if "userDefinedData" in indata:
 442             data = indata.pop("userDefinedData")
 443             if type(data) == dict:
 444                 indata["_admin"]["userDefinedData"] = data
 445             else:
 446                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 447                                       .format(type(data)),
 448                                       http_code=HTTPStatus.BAD_REQUEST)
 449
 450         if ("operationalState" in indata["_admin"] and
 451                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 452             raise EngineException("operationalState already {}".format(content["_admin"]["operationalState"]),
 453                                   http_code=HTTPStatus.CONFLICT)
 454
 455         return indata
 456
 457
 458 class VnfdTopic(DescriptorTopic):
 459     topic = "vnfds"
 460     topic_msg = "vnfd"
 461
 462     def __init__(self, db, fs, msg, auth):
 463         DescriptorTopic.__init__(self, db, fs, msg, auth)
 464
 465     def pyangbind_validation(self, item, data, force=False):
 466         if self._descriptor_data_is_in_old_format(data):
 467             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 468                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 469         try:
 470             myvnfd = etsi_nfv_vnfd.etsi_nfv_vnfd()
 471             pybindJSONDecoder.load_ietf_json({'etsi-nfv-vnfd:vnfd': data}, None, None, obj=myvnfd,
 472                                              path_helper=True, skip_unknown=force)
 473             out = pybindJSON.dumps(myvnfd, mode="ietf")
 474             desc_out = self._remove_envelop(yaml.safe_load(out))
 475             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 476             return deep_update_dict(data, desc_out)
 477         except Exception as e:
 478             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 479                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 480
 481     @staticmethod
 482     def _descriptor_data_is_in_old_format(data):
 483         return ('vnfd-catalog' in data) or ('vnfd:vnfd-catalog' in data)
 484
 485     @staticmethod
 486     def _remove_envelop(indata=None):
 487         if not indata:
 488             return {}
 489         clean_indata = indata
 490
 491         if clean_indata.get('etsi-nfv-vnfd:vnfd'):
 492             if not isinstance(clean_indata['etsi-nfv-vnfd:vnfd'], dict):
 493                 raise EngineException("'etsi-nfv-vnfd:vnfd' must be a dict")
 494             clean_indata = clean_indata['etsi-nfv-vnfd:vnfd']
 495         elif clean_indata.get('vnfd'):
 496             if not isinstance(clean_indata['vnfd'], dict):
 497                 raise EngineException("'vnfd' must be dict")
 498             clean_indata = clean_indata['vnfd']
 499
 500         return clean_indata
 501
 502     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 503         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
 504
 505         # set type of vnfd
 506         contains_pdu = False
 507         contains_vdu = False
 508         for vdu in get_iterable(final_content.get("vdu")):
 509             if vdu.get("pdu-type"):
 510                 contains_pdu = True
 511             else:
 512                 contains_vdu = True
 513         if contains_pdu:
 514             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 515         elif contains_vdu:
 516             final_content["_admin"]["type"] = "vnfd"
 517         # if neither vud nor pdu do not fill type
 518         return final_content
 519
 520     def check_conflict_on_del(self, session, _id, db_content):
 521         """
 522         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 523         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 524         that uses this vnfd
 525         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 526         :param _id: vnfd internal id
 527         :param db_content: The database content of the _id.
 528         :return: None or raises EngineException with the conflict
 529         """
 530         if session["force"]:
 531             return
 532         descriptor = db_content
 533         descriptor_id = descriptor.get("id")
 534         if not descriptor_id:  # empty vnfd not uploaded
 535             return
 536
 537         _filter = self._get_project_filter(session)
 538
 539         # check vnfrs using this vnfd
 540         _filter["vnfd-id"] = _id
 541         if self.db.get_list("vnfrs", _filter):
 542             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 543
 544         # check NSD referencing this VNFD
 545         del _filter["vnfd-id"]
 546         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 547         if self.db.get_list("nsds", _filter):
 548             raise EngineException("There is at least one NSD referencing this descriptor",
 549                                   http_code=HTTPStatus.CONFLICT)
 550
 551     def _validate_input_new(self, indata, storage_params, force=False):
 552         indata.pop("onboardingState", None)
 553         indata.pop("operationalState", None)
 554         indata.pop("usageState", None)
 555         indata.pop("links", None)
 556
 557         indata = self.pyangbind_validation("vnfds", indata, force)
 558         # Cross references validation in the descriptor
 559
 560         self.validate_mgmt_interface_connection_point(indata)
 561
 562         for vdu in get_iterable(indata.get("vdu")):
 563             self.validate_vdu_internal_connection_points(vdu)
 564             self._validate_vdu_charms_in_package(storage_params, vdu, indata)
 565             self._validate_vdu_cloud_init_in_package(storage_params, vdu, indata)
 566
 567         self._validate_vnf_charms_in_package(storage_params, indata)
 568
 569         self.validate_external_connection_points(indata)
 570         self.validate_internal_virtual_links(indata)
 571         self.validate_monitoring_params(indata)
 572         self.validate_scaling_group_descriptor(indata)
 573
 574         return indata
 575
 576     @staticmethod
 577     def validate_mgmt_interface_connection_point(indata):
 578         if not indata.get("vdu"):
 579             return
 580         if not indata.get("mgmt-cp"):
 581             raise EngineException("'mgmt-cp' is a mandatory field and it is not defined",
 582                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 583
 584         for cp in get_iterable(indata.get("ext-cpd")):
 585             if cp["id"] == indata["mgmt-cp"]:
 586                 break
 587         else:
 588             raise EngineException("mgmt-cp='{}' must match an existing ext-cpd".format(indata["mgmt-cp"]),
 589                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 590
 591     @staticmethod
 592     def validate_vdu_internal_connection_points(vdu):
 593         int_cpds = set()
 594         for cpd in get_iterable(vdu.get("int-cpd")):
 595             cpd_id = cpd.get("id")
 596             if cpd_id and cpd_id in int_cpds:
 597                 raise EngineException("vdu[id='{}']:int-cpd[id='{}'] is already used by other int-cpd"
 598                                       .format(vdu["id"], cpd_id),
 599                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 600             int_cpds.add(cpd_id)
 601
 602     @staticmethod
 603     def validate_external_connection_points(indata):
 604         all_vdus_int_cpds = set()
 605         for vdu in get_iterable(indata.get("vdu")):
 606             for int_cpd in get_iterable(vdu.get("int-cpd")):
 607                 all_vdus_int_cpds.add((vdu.get("id"), int_cpd.get("id")))
 608
 609         ext_cpds = set()
 610         for cpd in get_iterable(indata.get("ext-cpd")):
 611             cpd_id = cpd.get("id")
 612             if cpd_id and cpd_id in ext_cpds:
 613                 raise EngineException("ext-cpd[id='{}'] is already used by other ext-cpd".format(cpd_id),
 614                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 615             ext_cpds.add(cpd_id)
 616
 617             int_cpd = cpd.get("int-cpd")
 618             if int_cpd:
 619                 if (int_cpd.get("vdu-id"), int_cpd.get("cpd")) not in all_vdus_int_cpds:
 620                     raise EngineException("ext-cpd[id='{}']:int-cpd must match an existing vdu int-cpd".format(cpd_id),
 621                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 622             # TODO: Validate k8s-cluster-net points to a valid k8s-cluster:nets ?
 623
 624     def _validate_vdu_charms_in_package(self, storage_params, vdu, indata):
 625         if not vdu.get("vdu-configuration"):
 626             return
 627         for vdu_configuration in get_iterable(indata.get("vdu-configuration")):
 628             if vdu_configuration.get("juju"):
 629                 if not self._validate_package_folders(storage_params, 'charms'):
 630                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 631                                           "package".format(indata["id"]))
 632
 633     def _validate_vdu_cloud_init_in_package(self, storage_params, vdu, indata):
 634         if not vdu.get("cloud-init-file"):
 635             return
 636         if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 637             raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 638                                   "package".format(indata["id"], vdu["id"]))
 639
 640     def _validate_vnf_charms_in_package(self, storage_params, indata):
 641         if not indata.get("vnf-configuration"):
 642             return
 643         for vnf_configuration in get_iterable(indata.get("vnf-configuration")):
 644             if vnf_configuration.get("juju"):
 645                 if not self._validate_package_folders(storage_params, 'charms'):
 646                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 647                                           "package".format(indata["id"]))
 648
 649     def _validate_package_folders(self, storage_params, folder, file=None):
 650         if not storage_params or not storage_params.get("pkg-dir"):
 651             return False
 652         else:
 653             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 654                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 655             else:
 656                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 657             if file:
 658                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 659             else:
 660                 if self.fs.file_exists(f, 'dir'):
 661                     if self.fs.dir_ls(f):
 662                         return True
 663             return False
 664
 665     @staticmethod
 666     def validate_internal_virtual_links(indata):
 667         all_ivld_ids = set()
 668         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 669             ivld_id = ivld.get("id")
 670             if ivld_id and ivld_id in all_ivld_ids:
 671                 raise EngineException("Duplicated VLD id in int-virtual-link-desc[id={}]".format(ivld_id),
 672                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 673             else:
 674                 all_ivld_ids.add(ivld_id)
 675
 676         for vdu in get_iterable(indata.get("vdu")):
 677             for int_cpd in get_iterable(vdu.get("int-cpd")):
 678                 int_cpd_ivld_id = int_cpd.get("int-virtual-link-desc")
 679                 if int_cpd_ivld_id and int_cpd_ivld_id not in all_ivld_ids:
 680                     raise EngineException(
 681                         "vdu[id='{}']:int-cpd[id='{}']:int-virtual-link-desc='{}' must match an existing "
 682                         "int-virtual-link-desc".format(vdu["id"], int_cpd["id"], int_cpd_ivld_id),
 683                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 684
 685         for df in get_iterable(indata.get("df")):
 686             for vlp in get_iterable(df.get("virtual-link-profile")):
 687                 vlp_ivld_id = vlp.get("id")
 688                 if vlp_ivld_id and vlp_ivld_id not in all_ivld_ids:
 689                     raise EngineException("df[id='{}']:virtual-link-profile='{}' must match an existing "
 690                                           "int-virtual-link-desc".format(df["id"], vlp_ivld_id),
 691                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 692
 693     @staticmethod
 694     def validate_monitoring_params(indata):
 695         all_monitoring_params = set()
 696         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 697             for mp in get_iterable(ivld.get("monitoring-parameters")):
 698                 mp_id = mp.get("id")
 699                 if mp_id and mp_id in all_monitoring_params:
 700                     raise EngineException("Duplicated monitoring-parameter id in "
 701                                           "int-virtual-link-desc[id='{}']:monitoring-parameters[id='{}']"
 702                                           .format(ivld["id"], mp_id),
 703                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 704                 else:
 705                     all_monitoring_params.add(mp_id)
 706
 707         for vdu in get_iterable(indata.get("vdu")):
 708             for mp in get_iterable(vdu.get("monitoring-parameter")):
 709                 mp_id = mp.get("id")
 710                 if mp_id and mp_id in all_monitoring_params:
 711                     raise EngineException("Duplicated monitoring-parameter id in "
 712                                           "vdu[id='{}']:monitoring-parameter[id='{}']"
 713                                           .format(vdu["id"], mp_id),
 714                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 715                 else:
 716                     all_monitoring_params.add(mp_id)
 717
 718         for df in get_iterable(indata.get("df")):
 719             for mp in get_iterable(df.get("monitoring-parameter")):
 720                 mp_id = mp.get("id")
 721                 if mp_id and mp_id in all_monitoring_params:
 722                     raise EngineException("Duplicated monitoring-parameter id in "
 723                                           "df[id='{}']:monitoring-parameter[id='{}']"
 724                                           .format(df["id"], mp_id),
 725                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 726                 else:
 727                     all_monitoring_params.add(mp_id)
 728
 729     @staticmethod
 730     def validate_scaling_group_descriptor(indata):
 731         all_monitoring_params = set()
 732         for ivld in get_iterable(indata.get("int-virtual-link-desc")):
 733             for mp in get_iterable(ivld.get("monitoring-parameters")):
 734                 all_monitoring_params.add(mp.get("id"))
 735
 736         for vdu in get_iterable(indata.get("vdu")):
 737             for mp in get_iterable(vdu.get("monitoring-parameter")):
 738                 all_monitoring_params.add(mp.get("id"))
 739
 740         for df in get_iterable(indata.get("df")):
 741             for mp in get_iterable(df.get("monitoring-parameter")):
 742                 all_monitoring_params.add(mp.get("id"))
 743
 744         for df in get_iterable(indata.get("df")):
 745             for sa in get_iterable(df.get("scaling-aspect")):
 746                 for sp in get_iterable(sa.get("scaling-policy")):
 747                     for sc in get_iterable(sp.get("scaling-criteria")):
 748                         sc_monitoring_param = sc.get("vnf-monitoring-param-ref")
 749                         if sc_monitoring_param and sc_monitoring_param not in all_monitoring_params:
 750                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-policy"
 751                                                   "[name='{}']:scaling-criteria[name='{}']: "
 752                                                   "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 753                                                   .format(df["id"], sa["id"], sp["name"], sc["name"],
 754                                                           sc_monitoring_param),
 755                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 756
 757                 for sca in get_iterable(sa.get("scaling-config-action")):
 758                     if not indata.get("vnf-configuration"):
 759                         raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced "
 760                                               "by df[id='{}']:scaling-aspect[id='{}']:scaling-config-action"
 761                                               .format(df["id"], sa["id"]),
 762                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 763                     for configuration in get_iterable(indata["vnf-configuration"]):
 764                         for primitive in get_iterable(configuration.get("config-primitive")):
 765                             if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 766                                 break
 767                         else:
 768                             raise EngineException("df[id='{}']:scaling-aspect[id='{}']:scaling-config-action:vnf-"
 769                                                   "config-primitive-name-ref='{}' does not match any "
 770                                                   "vnf-configuration:config-primitive:name"
 771                                                   .format(df["id"], sa["id"], sca["vnf-config-primitive-name-ref"]),
 772                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 773
 774     def delete_extra(self, session, _id, db_content, not_send_msg=None):
 775         """
 776         Deletes associate file system storage (via super)
 777         Deletes associated vnfpkgops from database.
 778         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 779         :param _id: server internal id
 780         :param db_content: The database content of the descriptor
 781         :return: None
 782         :raises: FsException in case of error while deleting associated storage
 783         """
 784         super().delete_extra(session, _id, db_content, not_send_msg)
 785         self.db.del_list("vnfpkgops", {"vnfPkgId": _id})
 786
 787     def sol005_projection(self, data):
 788         data["onboardingState"] = data["_admin"]["onboardingState"]
 789         data["operationalState"] = data["_admin"]["operationalState"]
 790         data["usageState"] = data["_admin"]["usageState"]
 791
 792         links = {}
 793         links["self"] = {"href": "/vnfpkgm/v1/vnf_packages/{}".format(data["_id"])}
 794         links["vnfd"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(data["_id"])}
 795         links["packageContent"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/package_content".format(data["_id"])}
 796         data["_links"] = links
 797
 798         return super().sol005_projection(data)
 799
 800
 801 class NsdTopic(DescriptorTopic):
 802     topic = "nsds"
 803     topic_msg = "nsd"
 804
 805     def __init__(self, db, fs, msg, auth):
 806         DescriptorTopic.__init__(self, db, fs, msg, auth)
 807
 808     def pyangbind_validation(self, item, data, force=False):
 809         if self._descriptor_data_is_in_old_format(data):
 810             raise EngineException("ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
 811                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 812         try:
 813             nsd_vnf_profiles = data.get('df', [{}])[0].get('vnf-profile', [])
 814             mynsd = etsi_nfv_nsd.etsi_nfv_nsd()
 815             pybindJSONDecoder.load_ietf_json({'nsd': {'nsd': [data]}}, None, None, obj=mynsd,
 816                                              path_helper=True, skip_unknown=force)
 817             out = pybindJSON.dumps(mynsd, mode="ietf")
 818             desc_out = self._remove_envelop(yaml.safe_load(out))
 819             desc_out = self._remove_yang_prefixes_from_descriptor(desc_out)
 820             if nsd_vnf_profiles:
 821                 desc_out['df'][0]['vnf-profile'] = nsd_vnf_profiles
 822             return desc_out
 823         except Exception as e:
 824             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 825                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 826
 827     @staticmethod
 828     def _descriptor_data_is_in_old_format(data):
 829         return ('nsd-catalog' in data) or ('nsd:nsd-catalog' in data)
 830
 831     @staticmethod
 832     def _remove_envelop(indata=None):
 833         if not indata:
 834             return {}
 835         clean_indata = indata
 836
 837         if clean_indata.get('nsd'):
 838             clean_indata = clean_indata['nsd']
 839         elif clean_indata.get('etsi-nfv-nsd:nsd'):
 840             clean_indata = clean_indata['etsi-nfv-nsd:nsd']
 841         if clean_indata.get('nsd'):
 842             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 843                 raise EngineException("'nsd' must be a list of only one element")
 844             clean_indata = clean_indata['nsd'][0]
 845         return clean_indata
 846
 847     def _validate_input_new(self, indata, storage_params, force=False):
 848         indata.pop("nsdOnboardingState", None)
 849         indata.pop("nsdOperationalState", None)
 850         indata.pop("nsdUsageState", None)
 851
 852         indata.pop("links", None)
 853
 854         indata = self.pyangbind_validation("nsds", indata, force)
 855         # Cross references validation in the descriptor
 856         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 857         for vld in get_iterable(indata.get("virtual-link-desc")):
 858             self.validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata)
 859
 860         self.validate_vnf_profiles_vnfd_id(indata)
 861
 862         return indata
 863
 864     @staticmethod
 865     def validate_vld_mgmt_network_with_virtual_link_protocol_data(vld, indata):
 866         if not vld.get("mgmt-network"):
 867             return
 868         vld_id = vld.get("id")
 869         for df in get_iterable(indata.get("df")):
 870             for vlp in get_iterable(df.get("virtual-link-profile")):
 871                 if vld_id and vld_id == vlp.get("virtual-link-desc-id"):
 872                     if vlp.get("virtual-link-protocol-data"):
 873                         raise EngineException("Error at df[id='{}']:virtual-link-profile[id='{}']:virtual-link-"
 874                                               "protocol-data You cannot set a virtual-link-protocol-data "
 875                                               "when mgmt-network is True"
 876                                               .format(df["id"], vlp["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 877
 878     @staticmethod
 879     def validate_vnf_profiles_vnfd_id(indata):
 880         all_vnfd_ids = set(get_iterable(indata.get("vnfd-id")))
 881         for df in get_iterable(indata.get("df")):
 882             for vnf_profile in get_iterable(df.get("vnf-profile")):
 883                 vnfd_id = vnf_profile.get("vnfd-id")
 884                 if vnfd_id and vnfd_id not in all_vnfd_ids:
 885                     raise EngineException("Error at df[id='{}']:vnf_profile[id='{}']:vnfd-id='{}' "
 886                                           "does not match any vnfd-id".format(df["id"], vnf_profile["id"], vnfd_id),
 887                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 888
 889     def _validate_input_edit(self, indata, content, force=False):
 890         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 891         """
 892         indata looks as follows:
 893             - In the new case (conformant)
 894                 {'nsdOperationalState': 'DISABLED', 'userDefinedData': {'id': 'string23',
 895                 '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}}
 896             - In the old case (backwards-compatible)
 897                 {'id': 'string23', '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}
 898         """
 899         if "_admin" not in indata:
 900             indata["_admin"] = {}
 901
 902         if "nsdOperationalState" in indata:
 903             if indata["nsdOperationalState"] in ("ENABLED", "DISABLED"):
 904                 indata["_admin"]["operationalState"] = indata.pop("nsdOperationalState")
 905             else:
 906                 raise EngineException("State '{}' is not a valid operational state"
 907                                       .format(indata["nsdOperationalState"]),
 908                                       http_code=HTTPStatus.BAD_REQUEST)
 909
 910         # In the case of user defined data, we need to put the data in the root of the object
 911         # to preserve current expected behaviour
 912         if "userDefinedData" in indata:
 913             data = indata.pop("userDefinedData")
 914             if type(data) == dict:
 915                 indata["_admin"]["userDefinedData"] = data
 916             else:
 917                 raise EngineException("userDefinedData should be an object, but is '{}' instead"
 918                                       .format(type(data)),
 919                                       http_code=HTTPStatus.BAD_REQUEST)
 920         if ("operationalState" in indata["_admin"] and
 921                 content["_admin"]["operationalState"] == indata["_admin"]["operationalState"]):
 922             raise EngineException("nsdOperationalState already {}".format(content["_admin"]["operationalState"]),
 923                                   http_code=HTTPStatus.CONFLICT)
 924         return indata
 925
 926     def _check_descriptor_dependencies(self, session, descriptor):
 927         """
 928         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 929         connection points are ok
 930         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 931         :param descriptor: descriptor to be inserted or edit
 932         :return: None or raises exception
 933         """
 934         if session["force"]:
 935             return
 936         vnfds_index = self._get_descriptor_constituent_vnfds_index(session, descriptor)
 937
 938         # Cross references validation in the descriptor and vnfd connection point validation
 939         for df in get_iterable(descriptor.get("df")):
 940             self.validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index)
 941
 942     def _get_descriptor_constituent_vnfds_index(self, session, descriptor):
 943         vnfds_index = {}
 944         if descriptor.get("vnfd-id") and not session["force"]:
 945             for vnfd_id in get_iterable(descriptor.get("vnfd-id")):
 946                 query_filter = self._get_project_filter(session)
 947                 query_filter["id"] = vnfd_id
 948                 vnf_list = self.db.get_list("vnfds", query_filter)
 949                 if not vnf_list:
 950                     raise EngineException("Descriptor error at 'vnfd-id'='{}' references a non "
 951                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 952                 vnfds_index[vnfd_id] = vnf_list[0]
 953         return vnfds_index
 954
 955     @staticmethod
 956     def validate_df_vnf_profiles_constituent_connection_points(df, vnfds_index):
 957         for vnf_profile in get_iterable(df.get("vnf-profile")):
 958             vnfd = vnfds_index.get(vnf_profile["vnfd-id"])
 959             all_vnfd_ext_cpds = set()
 960             for ext_cpd in get_iterable(vnfd.get("ext-cpd")):
 961                 if ext_cpd.get('id'):
 962                     all_vnfd_ext_cpds.add(ext_cpd.get('id'))
 963
 964             for virtual_link in get_iterable(vnf_profile.get("virtual-link-connectivity")):
 965                 for vl_cpd in get_iterable(virtual_link.get("constituent-cpd-id")):
 966                     vl_cpd_id = vl_cpd.get('constituent-cpd-id')
 967                     if vl_cpd_id and vl_cpd_id not in all_vnfd_ext_cpds:
 968                         raise EngineException("Error at df[id='{}']:vnf-profile[id='{}']:virtual-link-connectivity"
 969                                               "[virtual-link-profile-id='{}']:constituent-cpd-id='{}' references a "
 970                                               "non existing ext-cpd:id inside vnfd '{}'"
 971                                               .format(df["id"], vnf_profile["id"],
 972                                                       virtual_link["virtual-link-profile-id"], vl_cpd_id, vnfd["id"]),
 973                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 974
 975     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 976         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
 977
 978         self._check_descriptor_dependencies(session, final_content)
 979
 980         return final_content
 981
 982     def check_conflict_on_del(self, session, _id, db_content):
 983         """
 984         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 985         that NSD can be public and be used by other projects.
 986         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 987         :param _id: nsd internal id
 988         :param db_content: The database content of the _id
 989         :return: None or raises EngineException with the conflict
 990         """
 991         if session["force"]:
 992             return
 993         descriptor = db_content
 994         descriptor_id = descriptor.get("id")
 995         if not descriptor_id:  # empty nsd not uploaded
 996             return
 997
 998         # check NSD used by NS
 999         _filter = self._get_project_filter(session)
1000         _filter["nsd-id"] = _id
1001         if self.db.get_list("nsrs", _filter):
1002             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
1003
1004         # check NSD referenced by NST
1005         del _filter["nsd-id"]
1006         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
1007         if self.db.get_list("nsts", _filter):
1008             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
1009                                   http_code=HTTPStatus.CONFLICT)
1010
1011     def sol005_projection(self, data):
1012         data["nsdOnboardingState"] = data["_admin"]["onboardingState"]
1013         data["nsdOperationalState"] = data["_admin"]["operationalState"]
1014         data["nsdUsageState"] = data["_admin"]["usageState"]
1015
1016         links = {}
1017         links["self"] = {"href": "/nsd/v1/ns_descriptors/{}".format(data["_id"])}
1018         links["nsd_content"] = {"href": "/nsd/v1/ns_descriptors/{}/nsd_content".format(data["_id"])}
1019         data["_links"] = links
1020
1021         return super().sol005_projection(data)
1022
1023
1024 class NstTopic(DescriptorTopic):
1025     topic = "nsts"
1026     topic_msg = "nst"
1027     quota_name = "slice_templates"
1028
1029     def __init__(self, db, fs, msg, auth):
1030         DescriptorTopic.__init__(self, db, fs, msg, auth)
1031
1032     def pyangbind_validation(self, item, data, force=False):
1033         try:
1034             mynst = nst_im()
1035             pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
1036                                              path_helper=True, skip_unknown=force)
1037             out = pybindJSON.dumps(mynst, mode="ietf")
1038             desc_out = self._remove_envelop(yaml.safe_load(out))
1039             return desc_out
1040         except Exception as e:
1041             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
1042                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
1043
1044     @staticmethod
1045     def _remove_envelop(indata=None):
1046         if not indata:
1047             return {}
1048         clean_indata = indata
1049
1050         if clean_indata.get('nst'):
1051             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
1052                 raise EngineException("'nst' must be a list only one element")
1053             clean_indata = clean_indata['nst'][0]
1054         elif clean_indata.get('nst:nst'):
1055             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
1056                 raise EngineException("'nst:nst' must be a list only one element")
1057             clean_indata = clean_indata['nst:nst'][0]
1058         return clean_indata
1059
1060     def _validate_input_new(self, indata, storage_params, force=False):
1061         indata.pop("onboardingState", None)
1062         indata.pop("operationalState", None)
1063         indata.pop("usageState", None)
1064         indata = self.pyangbind_validation("nsts", indata, force)
1065         return indata.copy()
1066
1067     def _check_descriptor_dependencies(self, session, descriptor):
1068         """
1069         Check that the dependent descriptors exist on a new descriptor or edition
1070         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1071         :param descriptor: descriptor to be inserted or edit
1072         :return: None or raises exception
1073         """
1074         if not descriptor.get("netslice-subnet"):
1075             return
1076         for nsd in descriptor["netslice-subnet"]:
1077             nsd_id = nsd["nsd-ref"]
1078             filter_q = self._get_project_filter(session)
1079             filter_q["id"] = nsd_id
1080             if not self.db.get_list("nsds", filter_q):
1081                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
1082                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
1083
1084     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
1085         final_content = super().check_conflict_on_edit(session, final_content, edit_content, _id)
1086
1087         self._check_descriptor_dependencies(session, final_content)
1088         return final_content
1089
1090     def check_conflict_on_del(self, session, _id, db_content):
1091         """
1092         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
1093         that NST can be public and be used by other projects.
1094         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1095         :param _id: nst internal id
1096         :param db_content: The database content of the _id.
1097         :return: None or raises EngineException with the conflict
1098         """
1099         # TODO: Check this method
1100         if session["force"]:
1101             return
1102         # Get Network Slice Template from Database
1103         _filter = self._get_project_filter(session)
1104         _filter["_admin.nst-id"] = _id
1105         if self.db.get_list("nsis", _filter):
1106             raise EngineException("there is at least one Netslice Instance using this descriptor",
1107                                   http_code=HTTPStatus.CONFLICT)
1108
1109     def sol005_projection(self, data):
1110         data["onboardingState"] = data["_admin"]["onboardingState"]
1111         data["operationalState"] = data["_admin"]["operationalState"]
1112         data["usageState"] = data["_admin"]["usageState"]
1113
1114         links = {}
1115         links["self"] = {"href": "/nst/v1/netslice_templates/{}".format(data["_id"])}
1116         links["nst"] = {"href": "/nst/v1/netslice_templates/{}/nst".format(data["_id"])}
1117         data["_links"] = links
1118
1119         return super().sol005_projection(data)
1120
1121
1122 class PduTopic(BaseTopic):
1123     topic = "pdus"
1124     topic_msg = "pdu"
1125     quota_name = "pduds"
1126     schema_new = pdu_new_schema
1127     schema_edit = pdu_edit_schema
1128
1129     def __init__(self, db, fs, msg, auth):
1130         BaseTopic.__init__(self, db, fs, msg, auth)
1131
1132     @staticmethod
1133     def format_on_new(content, project_id=None, make_public=False):
1134         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
1135         content["_admin"]["onboardingState"] = "CREATED"
1136         content["_admin"]["operationalState"] = "ENABLED"
1137         content["_admin"]["usageState"] = "NOT_IN_USE"
1138
1139     def check_conflict_on_del(self, session, _id, db_content):
1140         """
1141         Check that there is not any vnfr that uses this PDU
1142         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1143         :param _id: pdu internal id
1144         :param db_content: The database content of the _id.
1145         :return: None or raises EngineException with the conflict
1146         """
1147         if session["force"]:
1148             return
1149
1150         _filter = self._get_project_filter(session)
1151         _filter["vdur.pdu-id"] = _id
1152         if self.db.get_list("vnfrs", _filter):
1153             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)
1154
1155
1156 class VnfPkgOpTopic(BaseTopic):
1157     topic = "vnfpkgops"
1158     topic_msg = "vnfd"
1159     schema_new = vnfpkgop_new_schema
1160     schema_edit = None
1161
1162     def __init__(self, db, fs, msg, auth):
1163         BaseTopic.__init__(self, db, fs, msg, auth)
1164
1165     def edit(self, session, _id, indata=None, kwargs=None, content=None):
1166         raise EngineException("Method 'edit' not allowed for topic '{}'".format(self.topic),
1167                               HTTPStatus.METHOD_NOT_ALLOWED)
1168
1169     def delete(self, session, _id, dry_run=False):
1170         raise EngineException("Method 'delete' not allowed for topic '{}'".format(self.topic),
1171                               HTTPStatus.METHOD_NOT_ALLOWED)
1172
1173     def delete_list(self, session, filter_q=None):
1174         raise EngineException("Method 'delete_list' not allowed for topic '{}'".format(self.topic),
1175                               HTTPStatus.METHOD_NOT_ALLOWED)
1176
1177     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
1178         """
1179         Creates a new entry into database.
1180         :param rollback: list to append created items at database in case a rollback may to be done
1181         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1182         :param indata: data to be inserted
1183         :param kwargs: used to override the indata descriptor
1184         :param headers: http request headers
1185         :return: _id, op_id:
1186             _id: identity of the inserted data.
1187              op_id: None
1188         """
1189         self._update_input_with_kwargs(indata, kwargs)
1190         validate_input(indata, self.schema_new)
1191         vnfpkg_id = indata["vnfPkgId"]
1192         filter_q = BaseTopic._get_project_filter(session)
1193         filter_q["_id"] = vnfpkg_id
1194         vnfd = self.db.get_one("vnfds", filter_q)
1195         operation = indata["lcmOperationType"]
1196         kdu_name = indata["kdu_name"]
1197         for kdu in vnfd.get("kdu", []):
1198             if kdu["name"] == kdu_name:
1199                 helm_chart = kdu.get("helm-chart")
1200                 juju_bundle = kdu.get("juju-bundle")
1201                 break
1202         else:
1203             raise EngineException("Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id, kdu_name))
1204         if helm_chart:
1205             indata["helm-chart"] = helm_chart
1206             match = fullmatch(r"([^/]*)/([^/]*)", helm_chart)
1207             repo_name = match.group(1) if match else None
1208         elif juju_bundle:
1209             indata["juju-bundle"] = juju_bundle
1210             match = fullmatch(r"([^/]*)/([^/]*)", juju_bundle)
1211             repo_name = match.group(1) if match else None
1212         else:
1213             raise EngineException("Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']"
1214                                   .format(vnfpkg_id, kdu_name))
1215         if repo_name:
1216             del filter_q["_id"]
1217             filter_q["name"] = repo_name
1218             repo = self.db.get_one("k8srepos", filter_q)
1219             k8srepo_id = repo.get("_id")
1220             k8srepo_url = repo.get("url")
1221         else:
1222             k8srepo_id = None
1223             k8srepo_url = None
1224         indata["k8srepoId"] = k8srepo_id
1225         indata["k8srepo_url"] = k8srepo_url
1226         vnfpkgop_id = str(uuid4())
1227         vnfpkgop_desc = {
1228             "_id": vnfpkgop_id,
1229             "operationState": "PROCESSING",
1230             "vnfPkgId": vnfpkg_id,
1231             "lcmOperationType": operation,
1232             "isAutomaticInvocation": False,
1233             "isCancelPending": False,
1234             "operationParams": indata,
1235             "links": {
1236                 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id,
1237                 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id,
1238             }
1239         }
1240         self.format_on_new(vnfpkgop_desc, session["project_id"], make_public=session["public"])
1241         ctime = vnfpkgop_desc["_admin"]["created"]
1242         vnfpkgop_desc["statusEnteredTime"] = ctime
1243         vnfpkgop_desc["startTime"] = ctime
1244         self.db.create(self.topic, vnfpkgop_desc)
1245         rollback.append({"topic": self.topic, "_id": vnfpkgop_id})
1246         self.msg.write(self.topic_msg, operation, vnfpkgop_desc)
1247         return vnfpkgop_id, None