osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema
  24 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  25 from osm_im.vnfd import vnfd as vnfd_im
  26 from osm_im.nsd import nsd as nsd_im
  27 from osm_im.nst import nst as nst_im
  28 from pyangbind.lib.serialise import pybindJSONDecoder
  29 import pyangbind.lib.pybindJSON as pybindJSON
  30
  31 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  32
  33
  34 class DescriptorTopic(BaseTopic):
  35
  36     def __init__(self, db, fs, msg, auth):
  37         BaseTopic.__init__(self, db, fs, msg, auth)
  38
  39     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  40         super().check_conflict_on_edit(session, final_content, edit_content, _id)
  41         # 1. validate again with pyangbind
  42         # 1.1. remove internal keys
  43         internal_keys = {}
  44         for k in ("_id", "_admin"):
  45             if k in final_content:
  46                 internal_keys[k] = final_content.pop(k)
  47         storage_params = internal_keys["_admin"].get("storage")
  48         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  49         # 1.2. modify final_content with a serialized version
  50         final_content.clear()
  51         final_content.update(serialized)
  52         # 1.3. restore internal keys
  53         for k, v in internal_keys.items():
  54             final_content[k] = v
  55
  56         if session["force"]:
  57             return
  58         # 2. check that this id is not present
  59         if "id" in edit_content:
  60             _filter = self._get_project_filter(session)
  61             _filter["id"] = final_content["id"]
  62             _filter["_id.neq"] = _id
  63             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  64                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  65                                                                                                final_content["id"]),
  66                                       HTTPStatus.CONFLICT)
  67
  68     @staticmethod
  69     def format_on_new(content, project_id=None, make_public=False):
  70         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  71         content["_admin"]["onboardingState"] = "CREATED"
  72         content["_admin"]["operationalState"] = "DISABLED"
  73         content["_admin"]["usageState"] = "NOT_IN_USE"
  74
  75     def delete_extra(self, session, _id, db_content):
  76         """
  77         Deletes file system storage associated with the descriptor
  78         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
  79         :param _id: server internal id
  80         :param db_content: The database content of the descriptor
  81         :return: None if ok or raises EngineException with the problem
  82         """
  83         self.fs.file_delete(_id, ignore_non_exist=True)
  84         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
  85
  86     @staticmethod
  87     def get_one_by_id(db, session, topic, id):
  88         # find owned by this project
  89         _filter = BaseTopic._get_project_filter(session)
  90         _filter["id"] = id
  91         desc_list = db.get_list(topic, _filter)
  92         if len(desc_list) == 1:
  93             return desc_list[0]
  94         elif len(desc_list) > 1:
  95             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
  96                               HTTPStatus.CONFLICT)
  97
  98         # not found any: try to find public
  99         _filter = BaseTopic._get_project_filter(session)
 100         _filter["id"] = id
 101         desc_list = db.get_list(topic, _filter)
 102         if not desc_list:
 103             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 104         elif len(desc_list) == 1:
 105             return desc_list[0]
 106         else:
 107             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 108                 topic[:-1], id), HTTPStatus.CONFLICT)
 109
 110     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 111         """
 112         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 113         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 114         (self.upload_content)
 115         :param rollback: list to append created items at database in case a rollback may to be done
 116         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 117         :param indata: data to be inserted
 118         :param kwargs: used to override the indata descriptor
 119         :param headers: http request headers
 120         :return: _id, None: identity of the inserted data; and None as there is not any operation
 121         """
 122
 123         try:
 124             # Check Quota
 125             self.check_quota(session)
 126
 127             # _remove_envelop
 128             if indata:
 129                 if "userDefinedData" in indata:
 130                     indata = indata['userDefinedData']
 131
 132             # Override descriptor with query string kwargs
 133             self._update_input_with_kwargs(indata, kwargs)
 134             # uncomment when this method is implemented.
 135             # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 136             # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 137
 138             content = {"_admin": {"userDefinedData": indata}}
 139             self.format_on_new(content, session["project_id"], make_public=session["public"])
 140             _id = self.db.create(self.topic, content)
 141             rollback.append({"topic": self.topic, "_id": _id})
 142             return _id, None
 143         except ValidationError as e:
 144             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 145
 146     def upload_content(self, session, _id, indata, kwargs, headers):
 147         """
 148         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 149         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 150         :param _id : the nsd,vnfd is already created, this is the id
 151         :param indata: http body request
 152         :param kwargs: user query string to override parameters. NOT USED
 153         :param headers:  http request headers
 154         :return: True if package is completely uploaded or False if partial content has been uploded
 155             Raise exception on error
 156         """
 157         # Check that _id exists and it is valid
 158         current_desc = self.show(session, _id)
 159
 160         content_range_text = headers.get("Content-Range")
 161         expected_md5 = headers.get("Content-File-MD5")
 162         compressed = None
 163         content_type = headers.get("Content-Type")
 164         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 165                 "application/zip" in content_type:
 166             compressed = "gzip"
 167         filename = headers.get("Content-Filename")
 168         if not filename:
 169             filename = "package.tar.gz" if compressed else "package"
 170         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 171         file_pkg = None
 172         error_text = ""
 173         try:
 174             if content_range_text:
 175                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 176                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 177                     raise IndexError()
 178                 start = int(content_range[1])
 179                 end = int(content_range[2]) + 1
 180                 total = int(content_range[3])
 181             else:
 182                 start = 0
 183             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 184
 185             if start:
 186                 if not self.fs.file_exists(temp_folder, 'dir'):
 187                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 188             else:
 189                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 190                 self.fs.mkdir(temp_folder)
 191
 192             storage = self.fs.get_params()
 193             storage["folder"] = _id
 194
 195             file_path = (temp_folder, filename)
 196             if self.fs.file_exists(file_path, 'file'):
 197                 file_size = self.fs.file_size(file_path)
 198             else:
 199                 file_size = 0
 200             if file_size != start:
 201                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 202                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 203             file_pkg = self.fs.file_open(file_path, 'a+b')
 204             if isinstance(indata, dict):
 205                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 206                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 207             else:
 208                 indata_len = 0
 209                 while True:
 210                     indata_text = indata.read(4096)
 211                     indata_len += len(indata_text)
 212                     if not indata_text:
 213                         break
 214                     file_pkg.write(indata_text)
 215             if content_range_text:
 216                 if indata_len != end-start:
 217                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 218                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 219                 if end != total:
 220                     # TODO update to UPLOADING
 221                     return False
 222
 223             # PACKAGE UPLOADED
 224             if expected_md5:
 225                 file_pkg.seek(0, 0)
 226                 file_md5 = md5()
 227                 chunk_data = file_pkg.read(1024)
 228                 while chunk_data:
 229                     file_md5.update(chunk_data)
 230                     chunk_data = file_pkg.read(1024)
 231                 if expected_md5 != file_md5.hexdigest():
 232                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 233             file_pkg.seek(0, 0)
 234             if compressed == "gzip":
 235                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 236                 descriptor_file_name = None
 237                 for tarinfo in tar:
 238                     tarname = tarinfo.name
 239                     tarname_path = tarname.split("/")
 240                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 241                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 242                     if len(tarname_path) == 1 and not tarinfo.isdir():
 243                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 244                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 245                         storage["pkg-dir"] = tarname_path[0]
 246                         if len(tarname_path) == 2:
 247                             if descriptor_file_name:
 248                                 raise EngineException(
 249                                     "Found more than one descriptor file at package descriptor tar.gz")
 250                             descriptor_file_name = tarname
 251                 if not descriptor_file_name:
 252                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 253                 storage["descriptor"] = descriptor_file_name
 254                 storage["zipfile"] = filename
 255                 self.fs.file_extract(tar, temp_folder)
 256                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 257                     content = descriptor_file.read()
 258             else:
 259                 content = file_pkg.read()
 260                 storage["descriptor"] = descriptor_file_name = filename
 261
 262             if descriptor_file_name.endswith(".json"):
 263                 error_text = "Invalid json format "
 264                 indata = json.load(content)
 265             else:
 266                 error_text = "Invalid yaml format "
 267                 indata = yaml.load(content)
 268
 269             current_desc["_admin"]["storage"] = storage
 270             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 271             current_desc["_admin"]["operationalState"] = "ENABLED"
 272
 273             indata = self._remove_envelop(indata)
 274
 275             # Override descriptor with query string kwargs
 276             if kwargs:
 277                 self._update_input_with_kwargs(indata, kwargs)
 278             # it will call overrides method at VnfdTopic or NsdTopic
 279             # indata = self._validate_input_edit(indata, force=session["force"])
 280
 281             deep_update_rfc7396(current_desc, indata)
 282             self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 283             self.db.replace(self.topic, _id, current_desc)
 284             self.fs.dir_rename(temp_folder, _id)
 285
 286             indata["_id"] = _id
 287             self._send_msg("created", indata)
 288
 289             # TODO if descriptor has changed because kwargs update content and remove cached zip
 290             # TODO if zip is not present creates one
 291             return True
 292
 293         except EngineException:
 294             raise
 295         except IndexError:
 296             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 297                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 298         except IOError as e:
 299             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 300         except tarfile.ReadError as e:
 301             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 302         except (ValueError, yaml.YAMLError) as e:
 303             raise EngineException(error_text + str(e))
 304         except ValidationError as e:
 305             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 306         finally:
 307             if file_pkg:
 308                 file_pkg.close()
 309
 310     def get_file(self, session, _id, path=None, accept_header=None):
 311         """
 312         Return the file content of a vnfd or nsd
 313         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 314         :param _id: Identity of the vnfd, nsd
 315         :param path: artifact path or "$DESCRIPTOR" or None
 316         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 317         :return: opened file plus Accept format or raises an exception
 318         """
 319         accept_text = accept_zip = False
 320         if accept_header:
 321             if 'text/plain' in accept_header or '*/*' in accept_header:
 322                 accept_text = True
 323             if 'application/zip' in accept_header or '*/*' in accept_header:
 324                 accept_zip = 'application/zip'
 325             elif 'application/gzip' in accept_header:
 326                 accept_zip = 'application/gzip'
 327
 328         if not accept_text and not accept_zip:
 329             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 330                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 331
 332         content = self.show(session, _id)
 333         if content["_admin"]["onboardingState"] != "ONBOARDED":
 334             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 335                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 336                                   http_code=HTTPStatus.CONFLICT)
 337         storage = content["_admin"]["storage"]
 338         if path is not None and path != "$DESCRIPTOR":   # artifacts
 339             if not storage.get('pkg-dir'):
 340                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 341             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 342                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 343                 return folder_content, "text/plain"
 344                 # TODO manage folders in http
 345             else:
 346                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 347                     "application/octet-stream"
 348
 349         # pkgtype   accept  ZIP  TEXT    -> result
 350         # manyfiles         yes  X       -> zip
 351         #                   no   yes     -> error
 352         # onefile           yes  no      -> zip
 353         #                   X    yes     -> text
 354
 355         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 356             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 357         elif storage.get('pkg-dir') and not accept_zip:
 358             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 359                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 360         else:
 361             if not storage.get('zipfile'):
 362                 # TODO generate zipfile if not present
 363                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 364                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 365             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 366
 367     def pyangbind_validation(self, item, data, force=False):
 368         try:
 369             if item == "vnfds":
 370                 myvnfd = vnfd_im()
 371                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 372                                                  path_helper=True, skip_unknown=force)
 373                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 374             elif item == "nsds":
 375                 mynsd = nsd_im()
 376                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 377                                                  path_helper=True, skip_unknown=force)
 378                 out = pybindJSON.dumps(mynsd, mode="ietf")
 379             elif item == "nsts":
 380                 mynst = nst_im()
 381                 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
 382                                                  path_helper=True, skip_unknown=force)
 383                 out = pybindJSON.dumps(mynst, mode="ietf")
 384             else:
 385                 raise EngineException("Not possible to validate '{}' item".format(item),
 386                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 387
 388             desc_out = self._remove_envelop(yaml.safe_load(out))
 389             return desc_out
 390
 391         except Exception as e:
 392             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 393                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 394
 395
 396 class VnfdTopic(DescriptorTopic):
 397     topic = "vnfds"
 398     topic_msg = "vnfd"
 399
 400     def __init__(self, db, fs, msg, auth):
 401         DescriptorTopic.__init__(self, db, fs, msg, auth)
 402
 403     @staticmethod
 404     def _remove_envelop(indata=None):
 405         if not indata:
 406             return {}
 407         clean_indata = indata
 408         if clean_indata.get('vnfd:vnfd-catalog'):
 409             clean_indata = clean_indata['vnfd:vnfd-catalog']
 410         elif clean_indata.get('vnfd-catalog'):
 411             clean_indata = clean_indata['vnfd-catalog']
 412         if clean_indata.get('vnfd'):
 413             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 414                 raise EngineException("'vnfd' must be a list of only one element")
 415             clean_indata = clean_indata['vnfd'][0]
 416         elif clean_indata.get('vnfd:vnfd'):
 417             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 418                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 419             clean_indata = clean_indata['vnfd:vnfd'][0]
 420         return clean_indata
 421
 422     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 423         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 424
 425         # set type of vnfd
 426         contains_pdu = False
 427         contains_vdu = False
 428         for vdu in get_iterable(final_content.get("vdu")):
 429             if vdu.get("pdu-type"):
 430                 contains_pdu = True
 431             else:
 432                 contains_vdu = True
 433         if contains_pdu:
 434             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 435         elif contains_vdu:
 436             final_content["_admin"]["type"] = "vnfd"
 437         # if neither vud nor pdu do not fill type
 438
 439     def check_conflict_on_del(self, session, _id, db_content):
 440         """
 441         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 442         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 443         that uses this vnfd
 444         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 445         :param _id: vnfd internal id
 446         :param db_content: The database content of the _id.
 447         :return: None or raises EngineException with the conflict
 448         """
 449         if session["force"]:
 450             return
 451         descriptor = db_content
 452         descriptor_id = descriptor.get("id")
 453         if not descriptor_id:  # empty vnfd not uploaded
 454             return
 455
 456         _filter = self._get_project_filter(session)
 457
 458         # check vnfrs using this vnfd
 459         _filter["vnfd-id"] = _id
 460         if self.db.get_list("vnfrs", _filter):
 461             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 462
 463         # check NSD referencing this VNFD
 464         del _filter["vnfd-id"]
 465         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 466         if self.db.get_list("nsds", _filter):
 467             raise EngineException("There is at least one NSD referencing this descriptor",
 468                                   http_code=HTTPStatus.CONFLICT)
 469
 470     def _validate_input_new(self, indata, storage_params, force=False):
 471         indata = self.pyangbind_validation("vnfds", indata, force)
 472         # Cross references validation in the descriptor
 473         if indata.get("vdu"):
 474             if not indata.get("mgmt-interface"):
 475                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 476                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 477             if indata["mgmt-interface"].get("cp"):
 478                 for cp in get_iterable(indata.get("connection-point")):
 479                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 480                         break
 481                 else:
 482                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 483                                           .format(indata["mgmt-interface"]["cp"]),
 484                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 485
 486         for vdu in get_iterable(indata.get("vdu")):
 487             for interface in get_iterable(vdu.get("interface")):
 488                 if interface.get("external-connection-point-ref"):
 489                     for cp in get_iterable(indata.get("connection-point")):
 490                         if cp["name"] == interface["external-connection-point-ref"]:
 491                             break
 492                     else:
 493                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 494                                               "must match an existing connection-point"
 495                                               .format(vdu["id"], interface["name"],
 496                                                       interface["external-connection-point-ref"]),
 497                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 498
 499                 elif interface.get("internal-connection-point-ref"):
 500                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 501                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 502                             break
 503                     else:
 504                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 505                                               "must match an existing vdu:internal-connection-point"
 506                                               .format(vdu["id"], interface["name"],
 507                                                       interface["internal-connection-point-ref"]),
 508                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 509             # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 510             if vdu.get("vdu-configuration"):
 511                 if vdu["vdu-configuration"].get("juju"):
 512                     if not self._validate_package_folders(storage_params, 'charms'):
 513                         raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
 514                                               "package".format(indata["id"], vdu["id"]))
 515             # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
 516             if vdu.get("cloud-init-file"):
 517                 if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 518                     raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 519                                           "package".format(indata["id"], vdu["id"]))
 520         # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 521         if indata.get("vnf-configuration"):
 522             if indata["vnf-configuration"].get("juju"):
 523                 if not self._validate_package_folders(storage_params, 'charms'):
 524                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 525                                           "package".format(indata["id"]))
 526         vld_names = []  # For detection of duplicated VLD names
 527         for ivld in get_iterable(indata.get("internal-vld")):
 528             # BEGIN Detection of duplicated VLD names
 529             ivld_name = ivld["name"]
 530             if ivld_name in vld_names:
 531                 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
 532                                       .format(ivld["name"], indata["id"], ivld["id"]),
 533                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 534             else:
 535                 vld_names.append(ivld_name)
 536             # END Detection of duplicated VLD names
 537             for icp in get_iterable(ivld.get("internal-connection-point")):
 538                 icp_mark = False
 539                 for vdu in get_iterable(indata.get("vdu")):
 540                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 541                         if icp["id-ref"] == internal_cp["id"]:
 542                             icp_mark = True
 543                             break
 544                     if icp_mark:
 545                         break
 546                 else:
 547                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 548                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 549                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 550             if ivld.get("ip-profile-ref"):
 551                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 552                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 553                         break
 554                 else:
 555                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 556                         ivld["id"], ivld["ip-profile-ref"]),
 557                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 558         for mp in get_iterable(indata.get("monitoring-param")):
 559             if mp.get("vdu-monitoring-param"):
 560                 mp_vmp_mark = False
 561                 for vdu in get_iterable(indata.get("vdu")):
 562                     for vmp in get_iterable(vdu.get("monitoring-param")):
 563                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 564                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 565                             mp_vmp_mark = True
 566                             break
 567                     if mp_vmp_mark:
 568                         break
 569                 else:
 570                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 571                                           "defined at vdu[id='{}'] or vdu does not exist"
 572                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 573                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 574                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 575             elif mp.get("vdu-metric"):
 576                 mp_vm_mark = False
 577                 for vdu in get_iterable(indata.get("vdu")):
 578                     if vdu.get("vdu-configuration"):
 579                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 580                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 581                                     mp["vdu-metric"]["vdu-ref"]:
 582                                 mp_vm_mark = True
 583                                 break
 584                         if mp_vm_mark:
 585                             break
 586                 else:
 587                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 588                                           "vdu[id='{}'] or vdu does not exist"
 589                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 590                                                   mp["vdu-metric"]["vdu-ref"]),
 591                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 592
 593         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 594             for sp in get_iterable(sgd.get("scaling-policy")):
 595                 for sc in get_iterable(sp.get("scaling-criteria")):
 596                     for mp in get_iterable(indata.get("monitoring-param")):
 597                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 598                             break
 599                     else:
 600                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 601                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 602                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 603                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 604             for sgd_vdu in get_iterable(sgd.get("vdu")):
 605                 sgd_vdu_mark = False
 606                 for vdu in get_iterable(indata.get("vdu")):
 607                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 608                         sgd_vdu_mark = True
 609                         break
 610                 if sgd_vdu_mark:
 611                     break
 612             else:
 613                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 614                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 615                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 616             for sca in get_iterable(sgd.get("scaling-config-action")):
 617                 if not indata.get("vnf-configuration"):
 618                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 619                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 620                                           .format(sgd["name"]),
 621                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 622                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 623                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 624                         break
 625                 else:
 626                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 627                                           "primitive-name-ref='{}' does not match any "
 628                                           "vnf-configuration:config-primitive:name"
 629                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 630                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 631         return indata
 632
 633     def _validate_input_edit(self, indata, force=False):
 634         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 635         return indata
 636
 637     def _validate_package_folders(self, storage_params, folder, file=None):
 638         if not storage_params or not storage_params.get("pkg-dir"):
 639             return False
 640         else:
 641             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 642                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 643             else:
 644                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 645             if file:
 646                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 647             else:
 648                 if self.fs.file_exists(f, 'dir'):
 649                     if self.fs.dir_ls(f):
 650                         return True
 651             return False
 652
 653
 654 class NsdTopic(DescriptorTopic):
 655     topic = "nsds"
 656     topic_msg = "nsd"
 657
 658     def __init__(self, db, fs, msg, auth):
 659         DescriptorTopic.__init__(self, db, fs, msg, auth)
 660
 661     @staticmethod
 662     def _remove_envelop(indata=None):
 663         if not indata:
 664             return {}
 665         clean_indata = indata
 666
 667         if clean_indata.get('nsd:nsd-catalog'):
 668             clean_indata = clean_indata['nsd:nsd-catalog']
 669         elif clean_indata.get('nsd-catalog'):
 670             clean_indata = clean_indata['nsd-catalog']
 671         if clean_indata.get('nsd'):
 672             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 673                 raise EngineException("'nsd' must be a list of only one element")
 674             clean_indata = clean_indata['nsd'][0]
 675         elif clean_indata.get('nsd:nsd'):
 676             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 677                 raise EngineException("'nsd:nsd' must be a list of only one element")
 678             clean_indata = clean_indata['nsd:nsd'][0]
 679         return clean_indata
 680
 681     def _validate_input_new(self, indata, storage_params, force=False):
 682         indata = self.pyangbind_validation("nsds", indata, force)
 683         # Cross references validation in the descriptor
 684         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 685         for vld in get_iterable(indata.get("vld")):
 686             if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
 687                 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
 688                                       " You cannot set an ip-profile when mgmt-network is True"
 689                                       .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 690             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 691                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 692                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 693                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 694                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 695                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 696                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 697                                                                  constituent_vnfd["member-vnf-index"],
 698                                                                  constituent_vnfd["vnfd-id-ref"]),
 699                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 700                         break
 701                 else:
 702                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 703                                           "does not match any constituent-vnfd:member-vnf-index"
 704                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 705                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 706         return indata
 707
 708     def _validate_input_edit(self, indata, force=False):
 709         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 710         return indata
 711
 712     def _check_descriptor_dependencies(self, session, descriptor):
 713         """
 714         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 715         connection points are ok
 716         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 717         :param descriptor: descriptor to be inserted or edit
 718         :return: None or raises exception
 719         """
 720         if session["force"]:
 721             return
 722         member_vnfd_index = {}
 723         if descriptor.get("constituent-vnfd") and not session["force"]:
 724             for vnf in descriptor["constituent-vnfd"]:
 725                 vnfd_id = vnf["vnfd-id-ref"]
 726                 filter_q = self._get_project_filter(session)
 727                 filter_q["id"] = vnfd_id
 728                 vnf_list = self.db.get_list("vnfds", filter_q)
 729                 if not vnf_list:
 730                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 731                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 732                 # elif len(vnf_list) > 1:
 733                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 734                 #                           http_code=HTTPStatus.CONFLICT)
 735                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 736
 737         # Cross references validation in the descriptor and vnfd connection point validation
 738         for vld in get_iterable(descriptor.get("vld")):
 739             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 740                 # look if this vnfd contains this connection point
 741                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 742                 if not vnfd:
 743                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 744                                           "does not match any constituent-vnfd:member-vnf-index"
 745                                           .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
 746                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 747                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 748                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 749                         break
 750                 else:
 751                     raise EngineException(
 752                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 753                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 754                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 755                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 756                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 757
 758     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 759         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 760
 761         self._check_descriptor_dependencies(session, final_content)
 762
 763     def check_conflict_on_del(self, session, _id, db_content):
 764         """
 765         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 766         that NSD can be public and be used by other projects.
 767         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 768         :param _id: nsd internal id
 769         :param db_content: The database content of the _id
 770         :return: None or raises EngineException with the conflict
 771         """
 772         if session["force"]:
 773             return
 774         descriptor = db_content
 775         descriptor_id = descriptor.get("id")
 776         if not descriptor_id:  # empty nsd not uploaded
 777             return
 778
 779         # check NSD used by NS
 780         _filter = self._get_project_filter(session)
 781         _filter["nsd-id"] = _id
 782         if self.db.get_list("nsrs", _filter):
 783             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
 784
 785         # check NSD referenced by NST
 786         del _filter["nsd-id"]
 787         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
 788         if self.db.get_list("nsts", _filter):
 789             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
 790                                   http_code=HTTPStatus.CONFLICT)
 791
 792
 793 class NstTopic(DescriptorTopic):
 794     topic = "nsts"
 795     topic_msg = "nst"
 796
 797     def __init__(self, db, fs, msg, auth):
 798         DescriptorTopic.__init__(self, db, fs, msg, auth)
 799
 800     @staticmethod
 801     def _remove_envelop(indata=None):
 802         if not indata:
 803             return {}
 804         clean_indata = indata
 805
 806         if clean_indata.get('nst'):
 807             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 808                 raise EngineException("'nst' must be a list only one element")
 809             clean_indata = clean_indata['nst'][0]
 810         elif clean_indata.get('nst:nst'):
 811             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
 812                 raise EngineException("'nst:nst' must be a list only one element")
 813             clean_indata = clean_indata['nst:nst'][0]
 814         return clean_indata
 815
 816     def _validate_input_edit(self, indata, force=False):
 817         # TODO validate with pyangbind, serialize
 818         return indata
 819
 820     def _validate_input_new(self, indata, storage_params, force=False):
 821         indata = self.pyangbind_validation("nsts", indata, force)
 822         return indata.copy()
 823
 824     def _check_descriptor_dependencies(self, session, descriptor):
 825         """
 826         Check that the dependent descriptors exist on a new descriptor or edition
 827         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 828         :param descriptor: descriptor to be inserted or edit
 829         :return: None or raises exception
 830         """
 831         if not descriptor.get("netslice-subnet"):
 832             return
 833         for nsd in descriptor["netslice-subnet"]:
 834             nsd_id = nsd["nsd-ref"]
 835             filter_q = self._get_project_filter(session)
 836             filter_q["id"] = nsd_id
 837             if not self.db.get_list("nsds", filter_q):
 838                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
 839                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
 840
 841     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 842         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 843
 844         self._check_descriptor_dependencies(session, final_content)
 845
 846     def check_conflict_on_del(self, session, _id, db_content):
 847         """
 848         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
 849         that NST can be public and be used by other projects.
 850         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 851         :param _id: nst internal id
 852         :param db_content: The database content of the _id.
 853         :return: None or raises EngineException with the conflict
 854         """
 855         # TODO: Check this method
 856         if session["force"]:
 857             return
 858         # Get Network Slice Template from Database
 859         _filter = self._get_project_filter(session)
 860         _filter["_admin.nst-id"] = _id
 861         if self.db.get_list("nsis", _filter):
 862             raise EngineException("there is at least one Netslice Instance using this descriptor",
 863                                   http_code=HTTPStatus.CONFLICT)
 864
 865
 866 class PduTopic(BaseTopic):
 867     topic = "pdus"
 868     topic_msg = "pdu"
 869     schema_new = pdu_new_schema
 870     schema_edit = pdu_edit_schema
 871
 872     def __init__(self, db, fs, msg, auth):
 873         BaseTopic.__init__(self, db, fs, msg, auth)
 874
 875     @staticmethod
 876     def format_on_new(content, project_id=None, make_public=False):
 877         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 878         content["_admin"]["onboardingState"] = "CREATED"
 879         content["_admin"]["operationalState"] = "ENABLED"
 880         content["_admin"]["usageState"] = "NOT_IN_USE"
 881
 882     def check_conflict_on_del(self, session, _id, db_content):
 883         """
 884         Check that there is not any vnfr that uses this PDU
 885         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 886         :param _id: pdu internal id
 887         :param db_content: The database content of the _id.
 888         :return: None or raises EngineException with the conflict
 889         """
 890         if session["force"]:
 891             return
 892
 893         _filter = self._get_project_filter(session)
 894         _filter["vdur.pdu-id"] = _id
 895         if self.db.get_list("vnfrs", _filter):
 896             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)