osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from time import time
  24 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema
  25 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
  26 from osm_im.vnfd import vnfd as vnfd_im
  27 from osm_im.nsd import nsd as nsd_im
  28 from osm_im.nst import nst as nst_im
  29 from pyangbind.lib.serialise import pybindJSONDecoder
  30 import pyangbind.lib.pybindJSON as pybindJSON
  31
  32 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  33
  34
  35 class DescriptorTopic(BaseTopic):
  36
  37     def __init__(self, db, fs, msg, auth):
  38         BaseTopic.__init__(self, db, fs, msg, auth)
  39
  40     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  41         super().check_conflict_on_edit(session, final_content, edit_content, _id)
  42         # 1. validate again with pyangbind
  43         # 1.1. remove internal keys
  44         internal_keys = {}
  45         for k in ("_id", "_admin"):
  46             if k in final_content:
  47                 internal_keys[k] = final_content.pop(k)
  48         storage_params = internal_keys["_admin"].get("storage")
  49         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  50         # 1.2. modify final_content with a serialized version
  51         final_content.clear()
  52         final_content.update(serialized)
  53         # 1.3. restore internal keys
  54         for k, v in internal_keys.items():
  55             final_content[k] = v
  56
  57         if session["force"]:
  58             return
  59         # 2. check that this id is not present
  60         if "id" in edit_content:
  61             _filter = self._get_project_filter(session)
  62             _filter["id"] = final_content["id"]
  63             _filter["_id.neq"] = _id
  64             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  65                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  66                                                                                                final_content["id"]),
  67                                       HTTPStatus.CONFLICT)
  68
  69     @staticmethod
  70     def format_on_new(content, project_id=None, make_public=False):
  71         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  72         content["_admin"]["onboardingState"] = "CREATED"
  73         content["_admin"]["operationalState"] = "DISABLED"
  74         content["_admin"]["usageState"] = "NOT_IN_USE"
  75
  76     def delete_extra(self, session, _id, db_content):
  77         """
  78         Deletes file system storage associated with the descriptor
  79         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
  80         :param _id: server internal id
  81         :param db_content: The database content of the descriptor
  82         :return: None if ok or raises EngineException with the problem
  83         """
  84         self.fs.file_delete(_id, ignore_non_exist=True)
  85         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
  86
  87     @staticmethod
  88     def get_one_by_id(db, session, topic, id):
  89         # find owned by this project
  90         _filter = BaseTopic._get_project_filter(session)
  91         _filter["id"] = id
  92         desc_list = db.get_list(topic, _filter)
  93         if len(desc_list) == 1:
  94             return desc_list[0]
  95         elif len(desc_list) > 1:
  96             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
  97                               HTTPStatus.CONFLICT)
  98
  99         # not found any: try to find public
 100         _filter = BaseTopic._get_project_filter(session)
 101         _filter["id"] = id
 102         desc_list = db.get_list(topic, _filter)
 103         if not desc_list:
 104             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 105         elif len(desc_list) == 1:
 106             return desc_list[0]
 107         else:
 108             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 109                 topic[:-1], id), HTTPStatus.CONFLICT)
 110
 111     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 112         """
 113         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 114         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 115         (self.upload_content)
 116         :param rollback: list to append created items at database in case a rollback may to be done
 117         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 118         :param indata: data to be inserted
 119         :param kwargs: used to override the indata descriptor
 120         :param headers: http request headers
 121         :return: _id, None: identity of the inserted data; and None as there is not any operation
 122         """
 123
 124         try:
 125             # Check Quota
 126             self.check_quota(session)
 127
 128             # _remove_envelop
 129             if indata:
 130                 if "userDefinedData" in indata:
 131                     indata = indata['userDefinedData']
 132
 133             # Override descriptor with query string kwargs
 134             self._update_input_with_kwargs(indata, kwargs)
 135             # uncomment when this method is implemented.
 136             # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 137             # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 138
 139             content = {"_admin": {"userDefinedData": indata}}
 140             self.format_on_new(content, session["project_id"], make_public=session["public"])
 141             _id = self.db.create(self.topic, content)
 142             rollback.append({"topic": self.topic, "_id": _id})
 143             self._send_msg("created", {"_id": _id})
 144             return _id, None
 145         except ValidationError as e:
 146             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 147
 148     def upload_content(self, session, _id, indata, kwargs, headers):
 149         """
 150         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 151         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 152         :param _id : the nsd,vnfd is already created, this is the id
 153         :param indata: http body request
 154         :param kwargs: user query string to override parameters. NOT USED
 155         :param headers:  http request headers
 156         :return: True if package is completely uploaded or False if partial content has been uploded
 157             Raise exception on error
 158         """
 159         # Check that _id exists and it is valid
 160         current_desc = self.show(session, _id)
 161
 162         content_range_text = headers.get("Content-Range")
 163         expected_md5 = headers.get("Content-File-MD5")
 164         compressed = None
 165         content_type = headers.get("Content-Type")
 166         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 167                 "application/zip" in content_type:
 168             compressed = "gzip"
 169         filename = headers.get("Content-Filename")
 170         if not filename:
 171             filename = "package.tar.gz" if compressed else "package"
 172         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 173         file_pkg = None
 174         error_text = ""
 175         try:
 176             if content_range_text:
 177                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 178                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 179                     raise IndexError()
 180                 start = int(content_range[1])
 181                 end = int(content_range[2]) + 1
 182                 total = int(content_range[3])
 183             else:
 184                 start = 0
 185             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 186
 187             if start:
 188                 if not self.fs.file_exists(temp_folder, 'dir'):
 189                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 190             else:
 191                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 192                 self.fs.mkdir(temp_folder)
 193
 194             storage = self.fs.get_params()
 195             storage["folder"] = _id
 196
 197             file_path = (temp_folder, filename)
 198             if self.fs.file_exists(file_path, 'file'):
 199                 file_size = self.fs.file_size(file_path)
 200             else:
 201                 file_size = 0
 202             if file_size != start:
 203                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 204                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 205             file_pkg = self.fs.file_open(file_path, 'a+b')
 206             if isinstance(indata, dict):
 207                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 208                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 209             else:
 210                 indata_len = 0
 211                 while True:
 212                     indata_text = indata.read(4096)
 213                     indata_len += len(indata_text)
 214                     if not indata_text:
 215                         break
 216                     file_pkg.write(indata_text)
 217             if content_range_text:
 218                 if indata_len != end-start:
 219                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 220                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 221                 if end != total:
 222                     # TODO update to UPLOADING
 223                     return False
 224
 225             # PACKAGE UPLOADED
 226             if expected_md5:
 227                 file_pkg.seek(0, 0)
 228                 file_md5 = md5()
 229                 chunk_data = file_pkg.read(1024)
 230                 while chunk_data:
 231                     file_md5.update(chunk_data)
 232                     chunk_data = file_pkg.read(1024)
 233                 if expected_md5 != file_md5.hexdigest():
 234                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 235             file_pkg.seek(0, 0)
 236             if compressed == "gzip":
 237                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 238                 descriptor_file_name = None
 239                 for tarinfo in tar:
 240                     tarname = tarinfo.name
 241                     tarname_path = tarname.split("/")
 242                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 243                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 244                     if len(tarname_path) == 1 and not tarinfo.isdir():
 245                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 246                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 247                         storage["pkg-dir"] = tarname_path[0]
 248                         if len(tarname_path) == 2:
 249                             if descriptor_file_name:
 250                                 raise EngineException(
 251                                     "Found more than one descriptor file at package descriptor tar.gz")
 252                             descriptor_file_name = tarname
 253                 if not descriptor_file_name:
 254                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 255                 storage["descriptor"] = descriptor_file_name
 256                 storage["zipfile"] = filename
 257                 self.fs.file_extract(tar, temp_folder)
 258                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 259                     content = descriptor_file.read()
 260             else:
 261                 content = file_pkg.read()
 262                 storage["descriptor"] = descriptor_file_name = filename
 263
 264             if descriptor_file_name.endswith(".json"):
 265                 error_text = "Invalid json format "
 266                 indata = json.load(content)
 267             else:
 268                 error_text = "Invalid yaml format "
 269                 indata = yaml.load(content, Loader=yaml.SafeLoader)
 270
 271             current_desc["_admin"]["storage"] = storage
 272             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 273             current_desc["_admin"]["operationalState"] = "ENABLED"
 274
 275             indata = self._remove_envelop(indata)
 276
 277             # Override descriptor with query string kwargs
 278             if kwargs:
 279                 self._update_input_with_kwargs(indata, kwargs)
 280             # it will call overrides method at VnfdTopic or NsdTopic
 281             # indata = self._validate_input_edit(indata, force=session["force"])
 282
 283             deep_update_rfc7396(current_desc, indata)
 284             self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 285             current_desc["_admin"]["modified"] = time()
 286             self.db.replace(self.topic, _id, current_desc)
 287             self.fs.dir_rename(temp_folder, _id)
 288
 289             indata["_id"] = _id
 290             self._send_msg("edited", indata)
 291
 292             # TODO if descriptor has changed because kwargs update content and remove cached zip
 293             # TODO if zip is not present creates one
 294             return True
 295
 296         except EngineException:
 297             raise
 298         except IndexError:
 299             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 300                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 301         except IOError as e:
 302             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 303         except tarfile.ReadError as e:
 304             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 305         except (ValueError, yaml.YAMLError) as e:
 306             raise EngineException(error_text + str(e))
 307         except ValidationError as e:
 308             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 309         finally:
 310             if file_pkg:
 311                 file_pkg.close()
 312
 313     def get_file(self, session, _id, path=None, accept_header=None):
 314         """
 315         Return the file content of a vnfd or nsd
 316         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 317         :param _id: Identity of the vnfd, nsd
 318         :param path: artifact path or "$DESCRIPTOR" or None
 319         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 320         :return: opened file plus Accept format or raises an exception
 321         """
 322         accept_text = accept_zip = False
 323         if accept_header:
 324             if 'text/plain' in accept_header or '*/*' in accept_header:
 325                 accept_text = True
 326             if 'application/zip' in accept_header or '*/*' in accept_header:
 327                 accept_zip = 'application/zip'
 328             elif 'application/gzip' in accept_header:
 329                 accept_zip = 'application/gzip'
 330
 331         if not accept_text and not accept_zip:
 332             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 333                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 334
 335         content = self.show(session, _id)
 336         if content["_admin"]["onboardingState"] != "ONBOARDED":
 337             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 338                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 339                                   http_code=HTTPStatus.CONFLICT)
 340         storage = content["_admin"]["storage"]
 341         if path is not None and path != "$DESCRIPTOR":   # artifacts
 342             if not storage.get('pkg-dir'):
 343                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 344             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 345                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 346                 return folder_content, "text/plain"
 347                 # TODO manage folders in http
 348             else:
 349                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 350                     "application/octet-stream"
 351
 352         # pkgtype   accept  ZIP  TEXT    -> result
 353         # manyfiles         yes  X       -> zip
 354         #                   no   yes     -> error
 355         # onefile           yes  no      -> zip
 356         #                   X    yes     -> text
 357
 358         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 359             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 360         elif storage.get('pkg-dir') and not accept_zip:
 361             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 362                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 363         else:
 364             if not storage.get('zipfile'):
 365                 # TODO generate zipfile if not present
 366                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 367                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 368             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 369
 370     def pyangbind_validation(self, item, data, force=False):
 371         try:
 372             if item == "vnfds":
 373                 myvnfd = vnfd_im()
 374                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 375                                                  path_helper=True, skip_unknown=force)
 376                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 377             elif item == "nsds":
 378                 mynsd = nsd_im()
 379                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 380                                                  path_helper=True, skip_unknown=force)
 381                 out = pybindJSON.dumps(mynsd, mode="ietf")
 382             elif item == "nsts":
 383                 mynst = nst_im()
 384                 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
 385                                                  path_helper=True, skip_unknown=force)
 386                 out = pybindJSON.dumps(mynst, mode="ietf")
 387             else:
 388                 raise EngineException("Not possible to validate '{}' item".format(item),
 389                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 390
 391             desc_out = self._remove_envelop(yaml.safe_load(out))
 392             return desc_out
 393
 394         except Exception as e:
 395             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 396                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 397
 398
 399 class VnfdTopic(DescriptorTopic):
 400     topic = "vnfds"
 401     topic_msg = "vnfd"
 402
 403     def __init__(self, db, fs, msg, auth):
 404         DescriptorTopic.__init__(self, db, fs, msg, auth)
 405
 406     @staticmethod
 407     def _remove_envelop(indata=None):
 408         if not indata:
 409             return {}
 410         clean_indata = indata
 411         if clean_indata.get('vnfd:vnfd-catalog'):
 412             clean_indata = clean_indata['vnfd:vnfd-catalog']
 413         elif clean_indata.get('vnfd-catalog'):
 414             clean_indata = clean_indata['vnfd-catalog']
 415         if clean_indata.get('vnfd'):
 416             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 417                 raise EngineException("'vnfd' must be a list of only one element")
 418             clean_indata = clean_indata['vnfd'][0]
 419         elif clean_indata.get('vnfd:vnfd'):
 420             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 421                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 422             clean_indata = clean_indata['vnfd:vnfd'][0]
 423         return clean_indata
 424
 425     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 426         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 427
 428         # set type of vnfd
 429         contains_pdu = False
 430         contains_vdu = False
 431         for vdu in get_iterable(final_content.get("vdu")):
 432             if vdu.get("pdu-type"):
 433                 contains_pdu = True
 434             else:
 435                 contains_vdu = True
 436         if contains_pdu:
 437             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 438         elif contains_vdu:
 439             final_content["_admin"]["type"] = "vnfd"
 440         # if neither vud nor pdu do not fill type
 441
 442     def check_conflict_on_del(self, session, _id, db_content):
 443         """
 444         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 445         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 446         that uses this vnfd
 447         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 448         :param _id: vnfd internal id
 449         :param db_content: The database content of the _id.
 450         :return: None or raises EngineException with the conflict
 451         """
 452         if session["force"]:
 453             return
 454         descriptor = db_content
 455         descriptor_id = descriptor.get("id")
 456         if not descriptor_id:  # empty vnfd not uploaded
 457             return
 458
 459         _filter = self._get_project_filter(session)
 460
 461         # check vnfrs using this vnfd
 462         _filter["vnfd-id"] = _id
 463         if self.db.get_list("vnfrs", _filter):
 464             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 465
 466         # check NSD referencing this VNFD
 467         del _filter["vnfd-id"]
 468         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 469         if self.db.get_list("nsds", _filter):
 470             raise EngineException("There is at least one NSD referencing this descriptor",
 471                                   http_code=HTTPStatus.CONFLICT)
 472
 473     def _validate_input_new(self, indata, storage_params, force=False):
 474         indata = self.pyangbind_validation("vnfds", indata, force)
 475         # Cross references validation in the descriptor
 476         if indata.get("vdu"):
 477             if not indata.get("mgmt-interface"):
 478                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 479                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 480             if indata["mgmt-interface"].get("cp"):
 481                 for cp in get_iterable(indata.get("connection-point")):
 482                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 483                         break
 484                 else:
 485                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 486                                           .format(indata["mgmt-interface"]["cp"]),
 487                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 488
 489         for vdu in get_iterable(indata.get("vdu")):
 490             for interface in get_iterable(vdu.get("interface")):
 491                 if interface.get("external-connection-point-ref"):
 492                     for cp in get_iterable(indata.get("connection-point")):
 493                         if cp["name"] == interface["external-connection-point-ref"]:
 494                             break
 495                     else:
 496                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 497                                               "must match an existing connection-point"
 498                                               .format(vdu["id"], interface["name"],
 499                                                       interface["external-connection-point-ref"]),
 500                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 501
 502                 elif interface.get("internal-connection-point-ref"):
 503                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 504                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 505                             break
 506                     else:
 507                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 508                                               "must match an existing vdu:internal-connection-point"
 509                                               .format(vdu["id"], interface["name"],
 510                                                       interface["internal-connection-point-ref"]),
 511                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 512             # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 513             if vdu.get("vdu-configuration"):
 514                 if vdu["vdu-configuration"].get("juju"):
 515                     if not self._validate_package_folders(storage_params, 'charms'):
 516                         raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
 517                                               "package".format(indata["id"], vdu["id"]))
 518             # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
 519             if vdu.get("cloud-init-file"):
 520                 if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 521                     raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 522                                           "package".format(indata["id"], vdu["id"]))
 523         # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 524         if indata.get("vnf-configuration"):
 525             if indata["vnf-configuration"].get("juju"):
 526                 if not self._validate_package_folders(storage_params, 'charms'):
 527                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 528                                           "package".format(indata["id"]))
 529         vld_names = []  # For detection of duplicated VLD names
 530         for ivld in get_iterable(indata.get("internal-vld")):
 531             # BEGIN Detection of duplicated VLD names
 532             ivld_name = ivld["name"]
 533             if ivld_name in vld_names:
 534                 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
 535                                       .format(ivld["name"], indata["id"], ivld["id"]),
 536                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 537             else:
 538                 vld_names.append(ivld_name)
 539             # END Detection of duplicated VLD names
 540             for icp in get_iterable(ivld.get("internal-connection-point")):
 541                 icp_mark = False
 542                 for vdu in get_iterable(indata.get("vdu")):
 543                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 544                         if icp["id-ref"] == internal_cp["id"]:
 545                             icp_mark = True
 546                             break
 547                     if icp_mark:
 548                         break
 549                 else:
 550                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 551                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 552                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 553             if ivld.get("ip-profile-ref"):
 554                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 555                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 556                         break
 557                 else:
 558                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 559                         ivld["id"], ivld["ip-profile-ref"]),
 560                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 561         for mp in get_iterable(indata.get("monitoring-param")):
 562             if mp.get("vdu-monitoring-param"):
 563                 mp_vmp_mark = False
 564                 for vdu in get_iterable(indata.get("vdu")):
 565                     for vmp in get_iterable(vdu.get("monitoring-param")):
 566                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 567                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 568                             mp_vmp_mark = True
 569                             break
 570                     if mp_vmp_mark:
 571                         break
 572                 else:
 573                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 574                                           "defined at vdu[id='{}'] or vdu does not exist"
 575                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 576                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 577                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 578             elif mp.get("vdu-metric"):
 579                 mp_vm_mark = False
 580                 for vdu in get_iterable(indata.get("vdu")):
 581                     if vdu.get("vdu-configuration"):
 582                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 583                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 584                                     mp["vdu-metric"]["vdu-ref"]:
 585                                 mp_vm_mark = True
 586                                 break
 587                         if mp_vm_mark:
 588                             break
 589                 else:
 590                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 591                                           "vdu[id='{}'] or vdu does not exist"
 592                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 593                                                   mp["vdu-metric"]["vdu-ref"]),
 594                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 595
 596         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 597             for sp in get_iterable(sgd.get("scaling-policy")):
 598                 for sc in get_iterable(sp.get("scaling-criteria")):
 599                     for mp in get_iterable(indata.get("monitoring-param")):
 600                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 601                             break
 602                     else:
 603                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 604                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 605                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 606                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 607             for sgd_vdu in get_iterable(sgd.get("vdu")):
 608                 sgd_vdu_mark = False
 609                 for vdu in get_iterable(indata.get("vdu")):
 610                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 611                         sgd_vdu_mark = True
 612                         break
 613                 if sgd_vdu_mark:
 614                     break
 615             else:
 616                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 617                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 618                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 619             for sca in get_iterable(sgd.get("scaling-config-action")):
 620                 if not indata.get("vnf-configuration"):
 621                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 622                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 623                                           .format(sgd["name"]),
 624                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 625                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 626                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 627                         break
 628                 else:
 629                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 630                                           "primitive-name-ref='{}' does not match any "
 631                                           "vnf-configuration:config-primitive:name"
 632                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 633                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 634         return indata
 635
 636     def _validate_input_edit(self, indata, force=False):
 637         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 638         return indata
 639
 640     def _validate_package_folders(self, storage_params, folder, file=None):
 641         if not storage_params or not storage_params.get("pkg-dir"):
 642             return False
 643         else:
 644             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 645                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 646             else:
 647                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 648             if file:
 649                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 650             else:
 651                 if self.fs.file_exists(f, 'dir'):
 652                     if self.fs.dir_ls(f):
 653                         return True
 654             return False
 655
 656
 657 class NsdTopic(DescriptorTopic):
 658     topic = "nsds"
 659     topic_msg = "nsd"
 660
 661     def __init__(self, db, fs, msg, auth):
 662         DescriptorTopic.__init__(self, db, fs, msg, auth)
 663
 664     @staticmethod
 665     def _remove_envelop(indata=None):
 666         if not indata:
 667             return {}
 668         clean_indata = indata
 669
 670         if clean_indata.get('nsd:nsd-catalog'):
 671             clean_indata = clean_indata['nsd:nsd-catalog']
 672         elif clean_indata.get('nsd-catalog'):
 673             clean_indata = clean_indata['nsd-catalog']
 674         if clean_indata.get('nsd'):
 675             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 676                 raise EngineException("'nsd' must be a list of only one element")
 677             clean_indata = clean_indata['nsd'][0]
 678         elif clean_indata.get('nsd:nsd'):
 679             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 680                 raise EngineException("'nsd:nsd' must be a list of only one element")
 681             clean_indata = clean_indata['nsd:nsd'][0]
 682         return clean_indata
 683
 684     def _validate_input_new(self, indata, storage_params, force=False):
 685         indata = self.pyangbind_validation("nsds", indata, force)
 686         # Cross references validation in the descriptor
 687         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 688         for vld in get_iterable(indata.get("vld")):
 689             if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
 690                 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
 691                                       " You cannot set an ip-profile when mgmt-network is True"
 692                                       .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 693             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 694                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 695                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 696                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 697                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 698                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 699                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 700                                                                  constituent_vnfd["member-vnf-index"],
 701                                                                  constituent_vnfd["vnfd-id-ref"]),
 702                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 703                         break
 704                 else:
 705                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 706                                           "does not match any constituent-vnfd:member-vnf-index"
 707                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 708                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 709         # Check VNFFGD
 710         for fgd in get_iterable(indata.get("vnffgd")):
 711             for cls in get_iterable(fgd.get("classifier")):
 712                 rspref = cls.get("rsp-id-ref")
 713                 for rsp in get_iterable(fgd.get("rsp")):
 714                     rspid = rsp.get("id")
 715                     if rspid and rspref and rspid == rspref:
 716                         break
 717                 else:
 718                     raise EngineException(
 719                         "Error at vnffgd[id='{}']:classifier[id='{}']:rsp-id-ref '{}' does not match any rsp:id"
 720                         .format(fgd["id"], cls["id"], rspref),
 721                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 722         return indata
 723
 724     def _validate_input_edit(self, indata, force=False):
 725         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 726         return indata
 727
 728     def _check_descriptor_dependencies(self, session, descriptor):
 729         """
 730         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 731         connection points are ok
 732         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 733         :param descriptor: descriptor to be inserted or edit
 734         :return: None or raises exception
 735         """
 736         if session["force"]:
 737             return
 738         member_vnfd_index = {}
 739         if descriptor.get("constituent-vnfd") and not session["force"]:
 740             for vnf in descriptor["constituent-vnfd"]:
 741                 vnfd_id = vnf["vnfd-id-ref"]
 742                 filter_q = self._get_project_filter(session)
 743                 filter_q["id"] = vnfd_id
 744                 vnf_list = self.db.get_list("vnfds", filter_q)
 745                 if not vnf_list:
 746                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 747                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 748                 # elif len(vnf_list) > 1:
 749                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 750                 #                           http_code=HTTPStatus.CONFLICT)
 751                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 752
 753         # Cross references validation in the descriptor and vnfd connection point validation
 754         for vld in get_iterable(descriptor.get("vld")):
 755             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 756                 # look if this vnfd contains this connection point
 757                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 758                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 759                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 760                         break
 761                 else:
 762                     raise EngineException(
 763                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 764                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 765                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 766                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 767                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 768
 769     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 770         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 771
 772         self._check_descriptor_dependencies(session, final_content)
 773
 774     def check_conflict_on_del(self, session, _id, db_content):
 775         """
 776         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 777         that NSD can be public and be used by other projects.
 778         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 779         :param _id: nsd internal id
 780         :param db_content: The database content of the _id
 781         :return: None or raises EngineException with the conflict
 782         """
 783         if session["force"]:
 784             return
 785         descriptor = db_content
 786         descriptor_id = descriptor.get("id")
 787         if not descriptor_id:  # empty nsd not uploaded
 788             return
 789
 790         # check NSD used by NS
 791         _filter = self._get_project_filter(session)
 792         _filter["nsd-id"] = _id
 793         if self.db.get_list("nsrs", _filter):
 794             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
 795
 796         # check NSD referenced by NST
 797         del _filter["nsd-id"]
 798         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
 799         if self.db.get_list("nsts", _filter):
 800             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
 801                                   http_code=HTTPStatus.CONFLICT)
 802
 803
 804 class NstTopic(DescriptorTopic):
 805     topic = "nsts"
 806     topic_msg = "nst"
 807
 808     def __init__(self, db, fs, msg, auth):
 809         DescriptorTopic.__init__(self, db, fs, msg, auth)
 810
 811     @staticmethod
 812     def _remove_envelop(indata=None):
 813         if not indata:
 814             return {}
 815         clean_indata = indata
 816
 817         if clean_indata.get('nst'):
 818             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 819                 raise EngineException("'nst' must be a list only one element")
 820             clean_indata = clean_indata['nst'][0]
 821         elif clean_indata.get('nst:nst'):
 822             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
 823                 raise EngineException("'nst:nst' must be a list only one element")
 824             clean_indata = clean_indata['nst:nst'][0]
 825         return clean_indata
 826
 827     def _validate_input_edit(self, indata, force=False):
 828         # TODO validate with pyangbind, serialize
 829         return indata
 830
 831     def _validate_input_new(self, indata, storage_params, force=False):
 832         indata = self.pyangbind_validation("nsts", indata, force)
 833         return indata.copy()
 834
 835     def _check_descriptor_dependencies(self, session, descriptor):
 836         """
 837         Check that the dependent descriptors exist on a new descriptor or edition
 838         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 839         :param descriptor: descriptor to be inserted or edit
 840         :return: None or raises exception
 841         """
 842         if not descriptor.get("netslice-subnet"):
 843             return
 844         for nsd in descriptor["netslice-subnet"]:
 845             nsd_id = nsd["nsd-ref"]
 846             filter_q = self._get_project_filter(session)
 847             filter_q["id"] = nsd_id
 848             if not self.db.get_list("nsds", filter_q):
 849                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
 850                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
 851
 852     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 853         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 854
 855         self._check_descriptor_dependencies(session, final_content)
 856
 857     def check_conflict_on_del(self, session, _id, db_content):
 858         """
 859         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
 860         that NST can be public and be used by other projects.
 861         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 862         :param _id: nst internal id
 863         :param db_content: The database content of the _id.
 864         :return: None or raises EngineException with the conflict
 865         """
 866         # TODO: Check this method
 867         if session["force"]:
 868             return
 869         # Get Network Slice Template from Database
 870         _filter = self._get_project_filter(session)
 871         _filter["_admin.nst-id"] = _id
 872         if self.db.get_list("nsis", _filter):
 873             raise EngineException("there is at least one Netslice Instance using this descriptor",
 874                                   http_code=HTTPStatus.CONFLICT)
 875
 876
 877 class PduTopic(BaseTopic):
 878     topic = "pdus"
 879     topic_msg = "pdu"
 880     schema_new = pdu_new_schema
 881     schema_edit = pdu_edit_schema
 882
 883     def __init__(self, db, fs, msg, auth):
 884         BaseTopic.__init__(self, db, fs, msg, auth)
 885
 886     @staticmethod
 887     def format_on_new(content, project_id=None, make_public=False):
 888         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 889         content["_admin"]["onboardingState"] = "CREATED"
 890         content["_admin"]["operationalState"] = "ENABLED"
 891         content["_admin"]["usageState"] = "NOT_IN_USE"
 892
 893     def check_conflict_on_del(self, session, _id, db_content):
 894         """
 895         Check that there is not any vnfr that uses this PDU
 896         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 897         :param _id: pdu internal id
 898         :param db_content: The database content of the _id.
 899         :return: None or raises EngineException with the conflict
 900         """
 901         if session["force"]:
 902             return
 903
 904         _filter = self._get_project_filter(session)
 905         _filter["vdur.pdu-id"] = _id
 906         if self.db.get_list("vnfrs", _filter):
 907             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)