osm_nbi/descriptor_topics.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #    http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  12 # implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import tarfile
  17 import yaml
  18 import json
  19 # import logging
  20 from hashlib import md5
  21 from osm_common.dbbase import DbException, deep_update_rfc7396
  22 from http import HTTPStatus
  23 from validation import ValidationError, pdu_new_schema, pdu_edit_schema
  24 from base_topic import BaseTopic, EngineException, get_iterable
  25 from osm_im.vnfd import vnfd as vnfd_im
  26 from osm_im.nsd import nsd as nsd_im
  27 from osm_im.nst import nst as nst_im
  28 from pyangbind.lib.serialise import pybindJSONDecoder
  29 import pyangbind.lib.pybindJSON as pybindJSON
  30
  31 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  32
  33
  34 class DescriptorTopic(BaseTopic):
  35
  36     def __init__(self, db, fs, msg):
  37         BaseTopic.__init__(self, db, fs, msg)
  38
  39     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
  40         super().check_conflict_on_edit(session, final_content, edit_content, _id)
  41         # 1. validate again with pyangbind
  42         # 1.1. remove internal keys
  43         internal_keys = {}
  44         for k in ("_id", "_admin"):
  45             if k in final_content:
  46                 internal_keys[k] = final_content.pop(k)
  47         storage_params = internal_keys["_admin"].get("storage")
  48         serialized = self._validate_input_new(final_content, storage_params, session["force"])
  49         # 1.2. modify final_content with a serialized version
  50         final_content.clear()
  51         final_content.update(serialized)
  52         # 1.3. restore internal keys
  53         for k, v in internal_keys.items():
  54             final_content[k] = v
  55
  56         if session["force"]:
  57             return
  58         # 2. check that this id is not present
  59         if "id" in edit_content:
  60             _filter = self._get_project_filter(session)
  61             _filter["id"] = final_content["id"]
  62             _filter["_id.neq"] = _id
  63             if self.db.get_one(self.topic, _filter, fail_on_empty=False):
  64                 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
  65                                                                                                final_content["id"]),
  66                                       HTTPStatus.CONFLICT)
  67
  68     @staticmethod
  69     def format_on_new(content, project_id=None, make_public=False):
  70         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
  71         content["_admin"]["onboardingState"] = "CREATED"
  72         content["_admin"]["operationalState"] = "DISABLED"
  73         content["_admin"]["usageState"] = "NOT_IN_USE"
  74
  75     def delete_extra(self, session, _id, db_content):
  76         """
  77         Deletes file system storage associated with the descriptor
  78         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
  79         :param _id: server internal id
  80         :param db_content: The database content of the descriptor
  81         :return: None if ok or raises EngineException with the problem
  82         """
  83         self.fs.file_delete(_id, ignore_non_exist=True)
  84         self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
  85
  86     @staticmethod
  87     def get_one_by_id(db, session, topic, id):
  88         # find owned by this project
  89         _filter = BaseTopic._get_project_filter(session)
  90         _filter["id"] = id
  91         desc_list = db.get_list(topic, _filter)
  92         if len(desc_list) == 1:
  93             return desc_list[0]
  94         elif len(desc_list) > 1:
  95             raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
  96                               HTTPStatus.CONFLICT)
  97
  98         # not found any: try to find public
  99         _filter = BaseTopic._get_project_filter(session)
 100         _filter["id"] = id
 101         desc_list = db.get_list(topic, _filter)
 102         if not desc_list:
 103             raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
 104         elif len(desc_list) == 1:
 105             return desc_list[0]
 106         else:
 107             raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
 108                 topic[:-1], id), HTTPStatus.CONFLICT)
 109
 110     def new(self, rollback, session, indata=None, kwargs=None, headers=None):
 111         """
 112         Creates a new almost empty DISABLED  entry into database. Due to SOL005, it does not follow normal procedure.
 113         Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
 114         (self.upload_content)
 115         :param rollback: list to append created items at database in case a rollback may to be done
 116         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 117         :param indata: data to be inserted
 118         :param kwargs: used to override the indata descriptor
 119         :param headers: http request headers
 120         :return: _id: identity of the inserted data.
 121         """
 122
 123         try:
 124             # _remove_envelop
 125             if indata:
 126                 if "userDefinedData" in indata:
 127                     indata = indata['userDefinedData']
 128
 129             # Override descriptor with query string kwargs
 130             self._update_input_with_kwargs(indata, kwargs)
 131             # uncomment when this method is implemented.
 132             # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
 133             # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
 134
 135             content = {"_admin": {"userDefinedData": indata}}
 136             self.format_on_new(content, session["project_id"], make_public=session["public"])
 137             _id = self.db.create(self.topic, content)
 138             rollback.append({"topic": self.topic, "_id": _id})
 139             return _id
 140         except ValidationError as e:
 141             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 142
 143     def upload_content(self, session, _id, indata, kwargs, headers):
 144         """
 145         Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
 146         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 147         :param _id : the nsd,vnfd is already created, this is the id
 148         :param indata: http body request
 149         :param kwargs: user query string to override parameters. NOT USED
 150         :param headers:  http request headers
 151         :return: True if package is completely uploaded or False if partial content has been uploded
 152             Raise exception on error
 153         """
 154         # Check that _id exists and it is valid
 155         current_desc = self.show(session, _id)
 156
 157         content_range_text = headers.get("Content-Range")
 158         expected_md5 = headers.get("Content-File-MD5")
 159         compressed = None
 160         content_type = headers.get("Content-Type")
 161         if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
 162                 "application/zip" in content_type:
 163             compressed = "gzip"
 164         filename = headers.get("Content-Filename")
 165         if not filename:
 166             filename = "package.tar.gz" if compressed else "package"
 167         # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
 168         file_pkg = None
 169         error_text = ""
 170         try:
 171             if content_range_text:
 172                 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
 173                 if content_range[0] != "bytes":  # TODO check x<y not negative < total....
 174                     raise IndexError()
 175                 start = int(content_range[1])
 176                 end = int(content_range[2]) + 1
 177                 total = int(content_range[3])
 178             else:
 179                 start = 0
 180             temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 181
 182             if start:
 183                 if not self.fs.file_exists(temp_folder, 'dir'):
 184                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
 185             else:
 186                 self.fs.file_delete(temp_folder, ignore_non_exist=True)
 187                 self.fs.mkdir(temp_folder)
 188
 189             storage = self.fs.get_params()
 190             storage["folder"] = _id
 191
 192             file_path = (temp_folder, filename)
 193             if self.fs.file_exists(file_path, 'file'):
 194                 file_size = self.fs.file_size(file_path)
 195             else:
 196                 file_size = 0
 197             if file_size != start:
 198                 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
 199                     file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 200             file_pkg = self.fs.file_open(file_path, 'a+b')
 201             if isinstance(indata, dict):
 202                 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
 203                 file_pkg.write(indata_text.encode(encoding="utf-8"))
 204             else:
 205                 indata_len = 0
 206                 while True:
 207                     indata_text = indata.read(4096)
 208                     indata_len += len(indata_text)
 209                     if not indata_text:
 210                         break
 211                     file_pkg.write(indata_text)
 212             if content_range_text:
 213                 if indata_len != end-start:
 214                     raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
 215                         start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 216                 if end != total:
 217                     # TODO update to UPLOADING
 218                     return False
 219
 220             # PACKAGE UPLOADED
 221             if expected_md5:
 222                 file_pkg.seek(0, 0)
 223                 file_md5 = md5()
 224                 chunk_data = file_pkg.read(1024)
 225                 while chunk_data:
 226                     file_md5.update(chunk_data)
 227                     chunk_data = file_pkg.read(1024)
 228                 if expected_md5 != file_md5.hexdigest():
 229                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
 230             file_pkg.seek(0, 0)
 231             if compressed == "gzip":
 232                 tar = tarfile.open(mode='r', fileobj=file_pkg)
 233                 descriptor_file_name = None
 234                 for tarinfo in tar:
 235                     tarname = tarinfo.name
 236                     tarname_path = tarname.split("/")
 237                     if not tarname_path[0] or ".." in tarname_path:  # if start with "/" means absolute path
 238                         raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
 239                     if len(tarname_path) == 1 and not tarinfo.isdir():
 240                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
 241                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
 242                         storage["pkg-dir"] = tarname_path[0]
 243                         if len(tarname_path) == 2:
 244                             if descriptor_file_name:
 245                                 raise EngineException(
 246                                     "Found more than one descriptor file at package descriptor tar.gz")
 247                             descriptor_file_name = tarname
 248                 if not descriptor_file_name:
 249                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
 250                 storage["descriptor"] = descriptor_file_name
 251                 storage["zipfile"] = filename
 252                 self.fs.file_extract(tar, temp_folder)
 253                 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
 254                     content = descriptor_file.read()
 255             else:
 256                 content = file_pkg.read()
 257                 storage["descriptor"] = descriptor_file_name = filename
 258
 259             if descriptor_file_name.endswith(".json"):
 260                 error_text = "Invalid json format "
 261                 indata = json.load(content)
 262             else:
 263                 error_text = "Invalid yaml format "
 264                 indata = yaml.load(content)
 265
 266             current_desc["_admin"]["storage"] = storage
 267             current_desc["_admin"]["onboardingState"] = "ONBOARDED"
 268             current_desc["_admin"]["operationalState"] = "ENABLED"
 269
 270             indata = self._remove_envelop(indata)
 271
 272             # Override descriptor with query string kwargs
 273             if kwargs:
 274                 self._update_input_with_kwargs(indata, kwargs)
 275             # it will call overrides method at VnfdTopic or NsdTopic
 276             # indata = self._validate_input_edit(indata, force=session["force"])
 277
 278             deep_update_rfc7396(current_desc, indata)
 279             self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
 280             self.db.replace(self.topic, _id, current_desc)
 281             self.fs.dir_rename(temp_folder, _id)
 282
 283             indata["_id"] = _id
 284             self._send_msg("created", indata)
 285
 286             # TODO if descriptor has changed because kwargs update content and remove cached zip
 287             # TODO if zip is not present creates one
 288             return True
 289
 290         except EngineException:
 291             raise
 292         except IndexError:
 293             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
 294                                   HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
 295         except IOError as e:
 296             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
 297         except tarfile.ReadError as e:
 298             raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
 299         except (ValueError, yaml.YAMLError) as e:
 300             raise EngineException(error_text + str(e))
 301         except ValidationError as e:
 302             raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
 303         finally:
 304             if file_pkg:
 305                 file_pkg.close()
 306
 307     def get_file(self, session, _id, path=None, accept_header=None):
 308         """
 309         Return the file content of a vnfd or nsd
 310         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 311         :param _id: Identity of the vnfd, nsd
 312         :param path: artifact path or "$DESCRIPTOR" or None
 313         :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
 314         :return: opened file plus Accept format or raises an exception
 315         """
 316         accept_text = accept_zip = False
 317         if accept_header:
 318             if 'text/plain' in accept_header or '*/*' in accept_header:
 319                 accept_text = True
 320             if 'application/zip' in accept_header or '*/*' in accept_header:
 321                 accept_zip = 'application/zip'
 322             elif 'application/gzip' in accept_header:
 323                 accept_zip = 'application/gzip'
 324
 325         if not accept_text and not accept_zip:
 326             raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
 327                                   http_code=HTTPStatus.NOT_ACCEPTABLE)
 328
 329         content = self.show(session, _id)
 330         if content["_admin"]["onboardingState"] != "ONBOARDED":
 331             raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
 332                                   "onboardingState is {}".format(content["_admin"]["onboardingState"]),
 333                                   http_code=HTTPStatus.CONFLICT)
 334         storage = content["_admin"]["storage"]
 335         if path is not None and path != "$DESCRIPTOR":   # artifacts
 336             if not storage.get('pkg-dir'):
 337                 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
 338             if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
 339                 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
 340                 return folder_content, "text/plain"
 341                 # TODO manage folders in http
 342             else:
 343                 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
 344                     "application/octet-stream"
 345
 346         # pkgtype   accept  ZIP  TEXT    -> result
 347         # manyfiles         yes  X       -> zip
 348         #                   no   yes     -> error
 349         # onefile           yes  no      -> zip
 350         #                   X    yes     -> text
 351
 352         if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
 353             return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
 354         elif storage.get('pkg-dir') and not accept_zip:
 355             raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
 356                                   "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
 357         else:
 358             if not storage.get('zipfile'):
 359                 # TODO generate zipfile if not present
 360                 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
 361                                       "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
 362             return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
 363
 364     def pyangbind_validation(self, item, data, force=False):
 365         try:
 366             if item == "vnfds":
 367                 myvnfd = vnfd_im()
 368                 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
 369                                                  path_helper=True, skip_unknown=force)
 370                 out = pybindJSON.dumps(myvnfd, mode="ietf")
 371             elif item == "nsds":
 372                 mynsd = nsd_im()
 373                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
 374                                                  path_helper=True, skip_unknown=force)
 375                 out = pybindJSON.dumps(mynsd, mode="ietf")
 376             elif item == "nsts":
 377                 mynst = nst_im()
 378                 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
 379                                                  path_helper=True, skip_unknown=force)
 380                 out = pybindJSON.dumps(mynst, mode="ietf")
 381             else:
 382                 raise EngineException("Not possible to validate '{}' item".format(item),
 383                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 384
 385             desc_out = self._remove_envelop(yaml.safe_load(out))
 386             return desc_out
 387
 388         except Exception as e:
 389             raise EngineException("Error in pyangbind validation: {}".format(str(e)),
 390                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 391
 392
 393 class VnfdTopic(DescriptorTopic):
 394     topic = "vnfds"
 395     topic_msg = "vnfd"
 396
 397     def __init__(self, db, fs, msg):
 398         DescriptorTopic.__init__(self, db, fs, msg)
 399
 400     @staticmethod
 401     def _remove_envelop(indata=None):
 402         if not indata:
 403             return {}
 404         clean_indata = indata
 405         if clean_indata.get('vnfd:vnfd-catalog'):
 406             clean_indata = clean_indata['vnfd:vnfd-catalog']
 407         elif clean_indata.get('vnfd-catalog'):
 408             clean_indata = clean_indata['vnfd-catalog']
 409         if clean_indata.get('vnfd'):
 410             if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
 411                 raise EngineException("'vnfd' must be a list of only one element")
 412             clean_indata = clean_indata['vnfd'][0]
 413         elif clean_indata.get('vnfd:vnfd'):
 414             if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
 415                 raise EngineException("'vnfd:vnfd' must be a list of only one element")
 416             clean_indata = clean_indata['vnfd:vnfd'][0]
 417         return clean_indata
 418
 419     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 420         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 421
 422         # set type of vnfd
 423         contains_pdu = False
 424         contains_vdu = False
 425         for vdu in get_iterable(final_content.get("vdu")):
 426             if vdu.get("pdu-type"):
 427                 contains_pdu = True
 428             else:
 429                 contains_vdu = True
 430         if contains_pdu:
 431             final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
 432         elif contains_vdu:
 433             final_content["_admin"]["type"] = "vnfd"
 434         # if neither vud nor pdu do not fill type
 435
 436     def check_conflict_on_del(self, session, _id, db_content):
 437         """
 438         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
 439         that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
 440         that uses this vnfd
 441         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 442         :param _id: vnfd internal id
 443         :param db_content: The database content of the _id.
 444         :return: None or raises EngineException with the conflict
 445         """
 446         if session["force"]:
 447             return
 448         descriptor = db_content
 449         descriptor_id = descriptor.get("id")
 450         if not descriptor_id:  # empty vnfd not uploaded
 451             return
 452
 453         _filter = self._get_project_filter(session)
 454
 455         # check vnfrs using this vnfd
 456         _filter["vnfd-id"] = _id
 457         if self.db.get_list("vnfrs", _filter):
 458             raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
 459
 460         # check NSD referencing this VNFD
 461         del _filter["vnfd-id"]
 462         _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
 463         if self.db.get_list("nsds", _filter):
 464             raise EngineException("There is at least one NSD referencing this descriptor",
 465                                   http_code=HTTPStatus.CONFLICT)
 466
 467     def _validate_input_new(self, indata, storage_params, force=False):
 468         indata = self.pyangbind_validation("vnfds", indata, force)
 469         # Cross references validation in the descriptor
 470         if indata.get("vdu"):
 471             if not indata.get("mgmt-interface"):
 472                 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
 473                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 474             if indata["mgmt-interface"].get("cp"):
 475                 for cp in get_iterable(indata.get("connection-point")):
 476                     if cp["name"] == indata["mgmt-interface"]["cp"]:
 477                         break
 478                 else:
 479                     raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
 480                                           .format(indata["mgmt-interface"]["cp"]),
 481                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 482
 483         for vdu in get_iterable(indata.get("vdu")):
 484             for interface in get_iterable(vdu.get("interface")):
 485                 if interface.get("external-connection-point-ref"):
 486                     for cp in get_iterable(indata.get("connection-point")):
 487                         if cp["name"] == interface["external-connection-point-ref"]:
 488                             break
 489                     else:
 490                         raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
 491                                               "must match an existing connection-point"
 492                                               .format(vdu["id"], interface["name"],
 493                                                       interface["external-connection-point-ref"]),
 494                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 495
 496                 elif interface.get("internal-connection-point-ref"):
 497                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 498                         if interface["internal-connection-point-ref"] == internal_cp.get("id"):
 499                             break
 500                     else:
 501                         raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
 502                                               "must match an existing vdu:internal-connection-point"
 503                                               .format(vdu["id"], interface["name"],
 504                                                       interface["internal-connection-point-ref"]),
 505                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 506             # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 507             if vdu.get("vdu-configuration"):
 508                 if vdu["vdu-configuration"].get("juju"):
 509                     if not self._validate_package_folders(storage_params, 'charms'):
 510                         raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
 511                                               "package".format(indata["id"], vdu["id"]))
 512             # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
 513             if vdu.get("cloud-init-file"):
 514                 if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
 515                     raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
 516                                           "package".format(indata["id"], vdu["id"]))
 517         # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
 518         if indata.get("vnf-configuration"):
 519             if indata["vnf-configuration"].get("juju"):
 520                 if not self._validate_package_folders(storage_params, 'charms'):
 521                     raise EngineException("Charm defined in vnf[id={}] but not present in "
 522                                           "package".format(indata["id"]))
 523         vld_names = []  # For detection of duplicated VLD names
 524         for ivld in get_iterable(indata.get("internal-vld")):
 525             # BEGIN Detection of duplicated VLD names
 526             ivld_name = ivld["name"]
 527             if ivld_name in vld_names:
 528                 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
 529                                       .format(ivld["name"], indata["id"], ivld["id"]),
 530                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 531             else:
 532                 vld_names.append(ivld_name)
 533             # END Detection of duplicated VLD names
 534             for icp in get_iterable(ivld.get("internal-connection-point")):
 535                 icp_mark = False
 536                 for vdu in get_iterable(indata.get("vdu")):
 537                     for internal_cp in get_iterable(vdu.get("internal-connection-point")):
 538                         if icp["id-ref"] == internal_cp["id"]:
 539                             icp_mark = True
 540                             break
 541                     if icp_mark:
 542                         break
 543                 else:
 544                     raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
 545                                           "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
 546                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 547             if ivld.get("ip-profile-ref"):
 548                 for ip_prof in get_iterable(indata.get("ip-profiles")):
 549                     if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
 550                         break
 551                 else:
 552                     raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
 553                         ivld["id"], ivld["ip-profile-ref"]),
 554                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 555         for mp in get_iterable(indata.get("monitoring-param")):
 556             if mp.get("vdu-monitoring-param"):
 557                 mp_vmp_mark = False
 558                 for vdu in get_iterable(indata.get("vdu")):
 559                     for vmp in get_iterable(vdu.get("monitoring-param")):
 560                         if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
 561                                 mp["vdu-monitoring-param"]["vdu-ref"]:
 562                             mp_vmp_mark = True
 563                             break
 564                     if mp_vmp_mark:
 565                         break
 566                 else:
 567                     raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
 568                                           "defined at vdu[id='{}'] or vdu does not exist"
 569                                           .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
 570                                                   mp["vdu-monitoring-param"]["vdu-ref"]),
 571                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 572             elif mp.get("vdu-metric"):
 573                 mp_vm_mark = False
 574                 for vdu in get_iterable(indata.get("vdu")):
 575                     if vdu.get("vdu-configuration"):
 576                         for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
 577                             if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
 578                                     mp["vdu-metric"]["vdu-ref"]:
 579                                 mp_vm_mark = True
 580                                 break
 581                         if mp_vm_mark:
 582                             break
 583                 else:
 584                     raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
 585                                           "vdu[id='{}'] or vdu does not exist"
 586                                           .format(mp["vdu-metric"]["vdu-metric-name-ref"],
 587                                                   mp["vdu-metric"]["vdu-ref"]),
 588                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 589
 590         for sgd in get_iterable(indata.get("scaling-group-descriptor")):
 591             for sp in get_iterable(sgd.get("scaling-policy")):
 592                 for sc in get_iterable(sp.get("scaling-criteria")):
 593                     for mp in get_iterable(indata.get("monitoring-param")):
 594                         if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
 595                             break
 596                     else:
 597                         raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
 598                                               "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
 599                                               .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
 600                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 601             for sgd_vdu in get_iterable(sgd.get("vdu")):
 602                 sgd_vdu_mark = False
 603                 for vdu in get_iterable(indata.get("vdu")):
 604                     if vdu["id"] == sgd_vdu["vdu-id-ref"]:
 605                         sgd_vdu_mark = True
 606                         break
 607                 if sgd_vdu_mark:
 608                     break
 609             else:
 610                 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
 611                                       .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
 612                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 613             for sca in get_iterable(sgd.get("scaling-config-action")):
 614                 if not indata.get("vnf-configuration"):
 615                     raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
 616                                           "scaling-group-descriptor[name='{}']:scaling-config-action"
 617                                           .format(sgd["name"]),
 618                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 619                 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
 620                     if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
 621                         break
 622                 else:
 623                     raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
 624                                           "primitive-name-ref='{}' does not match any "
 625                                           "vnf-configuration:config-primitive:name"
 626                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
 627                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 628         return indata
 629
 630     def _validate_input_edit(self, indata, force=False):
 631         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 632         return indata
 633
 634     def _validate_package_folders(self, storage_params, folder, file=None):
 635         if not storage_params or not storage_params.get("pkg-dir"):
 636             return False
 637         else:
 638             if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
 639                 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 640             else:
 641                 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
 642             if file:
 643                 return self.fs.file_exists("{}/{}".format(f, file), 'file')
 644             else:
 645                 if self.fs.file_exists(f, 'dir'):
 646                     if self.fs.dir_ls(f):
 647                         return True
 648             return False
 649
 650
 651 class NsdTopic(DescriptorTopic):
 652     topic = "nsds"
 653     topic_msg = "nsd"
 654
 655     def __init__(self, db, fs, msg):
 656         DescriptorTopic.__init__(self, db, fs, msg)
 657
 658     @staticmethod
 659     def _remove_envelop(indata=None):
 660         if not indata:
 661             return {}
 662         clean_indata = indata
 663
 664         if clean_indata.get('nsd:nsd-catalog'):
 665             clean_indata = clean_indata['nsd:nsd-catalog']
 666         elif clean_indata.get('nsd-catalog'):
 667             clean_indata = clean_indata['nsd-catalog']
 668         if clean_indata.get('nsd'):
 669             if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
 670                 raise EngineException("'nsd' must be a list of only one element")
 671             clean_indata = clean_indata['nsd'][0]
 672         elif clean_indata.get('nsd:nsd'):
 673             if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
 674                 raise EngineException("'nsd:nsd' must be a list of only one element")
 675             clean_indata = clean_indata['nsd:nsd'][0]
 676         return clean_indata
 677
 678     def _validate_input_new(self, indata, storage_params, force=False):
 679         indata = self.pyangbind_validation("nsds", indata, force)
 680         # Cross references validation in the descriptor
 681         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
 682         for vld in get_iterable(indata.get("vld")):
 683             if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
 684                 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
 685                                       " You cannot set an ip-profile when mgmt-network is True"
 686                                       .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 687             for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 688                 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
 689                     if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
 690                         if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
 691                             raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
 692                                                   "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
 693                                                   " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
 694                                                                  constituent_vnfd["member-vnf-index"],
 695                                                                  constituent_vnfd["vnfd-id-ref"]),
 696                                                   http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 697                         break
 698                 else:
 699                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 700                                           "does not match any constituent-vnfd:member-vnf-index"
 701                                           .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
 702                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 703         return indata
 704
 705     def _validate_input_edit(self, indata, force=False):
 706         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
 707         return indata
 708
 709     def _check_descriptor_dependencies(self, session, descriptor):
 710         """
 711         Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
 712         connection points are ok
 713         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 714         :param descriptor: descriptor to be inserted or edit
 715         :return: None or raises exception
 716         """
 717         if session["force"]:
 718             return
 719         member_vnfd_index = {}
 720         if descriptor.get("constituent-vnfd") and not session["force"]:
 721             for vnf in descriptor["constituent-vnfd"]:
 722                 vnfd_id = vnf["vnfd-id-ref"]
 723                 filter_q = self._get_project_filter(session)
 724                 filter_q["id"] = vnfd_id
 725                 vnf_list = self.db.get_list("vnfds", filter_q)
 726                 if not vnf_list:
 727                     raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
 728                                           "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
 729                 # elif len(vnf_list) > 1:
 730                 #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
 731                 #                           http_code=HTTPStatus.CONFLICT)
 732                 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
 733
 734         # Cross references validation in the descriptor and vnfd connection point validation
 735         for vld in get_iterable(descriptor.get("vld")):
 736             for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
 737                 # look if this vnfd contains this connection point
 738                 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
 739                 if not vnfd:
 740                     raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
 741                                           "does not match any constituent-vnfd:member-vnf-index"
 742                                           .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
 743                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 744                 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
 745                     if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
 746                         break
 747                 else:
 748                     raise EngineException(
 749                         "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
 750                         "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
 751                         .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
 752                                 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
 753                         http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 754
 755     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 756         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 757
 758         self._check_descriptor_dependencies(session, final_content)
 759
 760     def check_conflict_on_del(self, session, _id, db_content):
 761         """
 762         Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
 763         that NSD can be public and be used by other projects.
 764         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 765         :param _id: nsd internal id
 766         :param db_content: The database content of the _id
 767         :return: None or raises EngineException with the conflict
 768         """
 769         if session["force"]:
 770             return
 771         descriptor = db_content
 772         descriptor_id = descriptor.get("id")
 773         if not descriptor_id:  # empty nsd not uploaded
 774             return
 775
 776         # check NSD used by NS
 777         _filter = self._get_project_filter(session)
 778         _filter["nsd-id"] = _id
 779         if self.db.get_list("nsrs", _filter):
 780             raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
 781
 782         # check NSD referenced by NST
 783         del _filter["nsd-id"]
 784         _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
 785         if self.db.get_list("nsts", _filter):
 786             raise EngineException("There is at least one NetSlice Template referencing this descriptor",
 787                                   http_code=HTTPStatus.CONFLICT)
 788
 789
 790 class NstTopic(DescriptorTopic):
 791     topic = "nsts"
 792     topic_msg = "nst"
 793
 794     def __init__(self, db, fs, msg):
 795         DescriptorTopic.__init__(self, db, fs, msg)
 796
 797     @staticmethod
 798     def _remove_envelop(indata=None):
 799         if not indata:
 800             return {}
 801         clean_indata = indata
 802
 803         if clean_indata.get('nst'):
 804             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
 805                 raise EngineException("'nst' must be a list only one element")
 806             clean_indata = clean_indata['nst'][0]
 807         elif clean_indata.get('nst:nst'):
 808             if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
 809                 raise EngineException("'nst:nst' must be a list only one element")
 810             clean_indata = clean_indata['nst:nst'][0]
 811         return clean_indata
 812
 813     def _validate_input_edit(self, indata, force=False):
 814         # TODO validate with pyangbind, serialize
 815         return indata
 816
 817     def _validate_input_new(self, indata, storage_params, force=False):
 818         indata = self.pyangbind_validation("nsts", indata, force)
 819         return indata.copy()
 820
 821     def _check_descriptor_dependencies(self, session, descriptor):
 822         """
 823         Check that the dependent descriptors exist on a new descriptor or edition
 824         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 825         :param descriptor: descriptor to be inserted or edit
 826         :return: None or raises exception
 827         """
 828         if not descriptor.get("netslice-subnet"):
 829             return
 830         for nsd in descriptor["netslice-subnet"]:
 831             nsd_id = nsd["nsd-ref"]
 832             filter_q = self._get_project_filter(session)
 833             filter_q["id"] = nsd_id
 834             if not self.db.get_list("nsds", filter_q):
 835                 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
 836                                       "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
 837
 838     def check_conflict_on_edit(self, session, final_content, edit_content, _id):
 839         super().check_conflict_on_edit(session, final_content, edit_content, _id)
 840
 841         self._check_descriptor_dependencies(session, final_content)
 842
 843     def check_conflict_on_del(self, session, _id, db_content):
 844         """
 845         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
 846         that NST can be public and be used by other projects.
 847         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 848         :param _id: nst internal id
 849         :param db_content: The database content of the _id.
 850         :return: None or raises EngineException with the conflict
 851         """
 852         # TODO: Check this method
 853         if session["force"]:
 854             return
 855         # Get Network Slice Template from Database
 856         _filter = self._get_project_filter(session)
 857         _filter["nst-id"] = _id
 858         if self.db.get_list("nsis", _filter):
 859             raise EngineException("there is at least one Netslice Instance using this descriptor",
 860                                   http_code=HTTPStatus.CONFLICT)
 861
 862
 863 class PduTopic(BaseTopic):
 864     topic = "pdus"
 865     topic_msg = "pdu"
 866     schema_new = pdu_new_schema
 867     schema_edit = pdu_edit_schema
 868
 869     def __init__(self, db, fs, msg):
 870         BaseTopic.__init__(self, db, fs, msg)
 871
 872     @staticmethod
 873     def format_on_new(content, project_id=None, make_public=False):
 874         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
 875         content["_admin"]["onboardingState"] = "CREATED"
 876         content["_admin"]["operationalState"] = "ENABLED"
 877         content["_admin"]["usageState"] = "NOT_IN_USE"
 878
 879     def check_conflict_on_del(self, session, _id, db_content):
 880         """
 881         Check that there is not any vnfr that uses this PDU
 882         :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
 883         :param _id: pdu internal id
 884         :param db_content: The database content of the _id.
 885         :return: None or raises EngineException with the conflict
 886         """
 887         if session["force"]:
 888             return
 889
 890         _filter = self._get_project_filter(session)
 891         _filter["vdur.pdu-id"] = _id
 892         if self.db.get_list("vnfrs", _filter):
 893             raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)