1 # -*- coding: utf-8 -*-
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
20 from hashlib
import md5
21 from osm_common
.dbbase
import DbException
, deep_update_rfc7396
22 from http
import HTTPStatus
24 from uuid
import uuid4
25 from re
import fullmatch
26 from osm_nbi
.validation
import ValidationError
, pdu_new_schema
, pdu_edit_schema
, \
27 validate_input
, vnfpkgop_new_schema
28 from osm_nbi
.base_topic
import BaseTopic
, EngineException
, get_iterable
29 from osm_im
.vnfd
import vnfd
as vnfd_im
30 from osm_im
.nsd
import nsd
as nsd_im
31 from osm_im
.nst
import nst
as nst_im
32 from pyangbind
.lib
.serialise
import pybindJSONDecoder
33 import pyangbind
.lib
.pybindJSON
as pybindJSON
35 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
38 class DescriptorTopic(BaseTopic
):
40 def __init__(self
, db
, fs
, msg
, auth
):
41 BaseTopic
.__init
__(self
, db
, fs
, msg
, auth
)
43 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
44 super().check_conflict_on_edit(session
, final_content
, edit_content
, _id
)
46 def _check_unique_id_name(descriptor
, position
=""):
47 for desc_key
, desc_item
in descriptor
.items():
48 if isinstance(desc_item
, list) and desc_item
:
51 for index
, list_item
in enumerate(desc_item
):
52 if isinstance(list_item
, dict):
53 _check_unique_id_name(list_item
, "{}.{}[{}]"
54 .format(position
, desc_key
, index
))
56 if index
== 0 and (list_item
.get("id") or list_item
.get("name")):
57 desc_item_id
= "id" if list_item
.get("id") else "name"
58 if desc_item_id
and list_item
.get(desc_item_id
):
59 if list_item
[desc_item_id
] in used_ids
:
60 position
= "{}.{}[{}]".format(position
, desc_key
, index
)
61 raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
62 .format(desc_item_id
, list_item
[desc_item_id
],
63 position
), HTTPStatus
.UNPROCESSABLE_ENTITY
)
64 used_ids
.append(list_item
[desc_item_id
])
65 _check_unique_id_name(final_content
)
66 # 1. validate again with pyangbind
67 # 1.1. remove internal keys
69 for k
in ("_id", "_admin"):
70 if k
in final_content
:
71 internal_keys
[k
] = final_content
.pop(k
)
72 storage_params
= internal_keys
["_admin"].get("storage")
73 serialized
= self
._validate
_input
_new
(final_content
, storage_params
, session
["force"])
74 # 1.2. modify final_content with a serialized version
76 final_content
.update(serialized
)
77 # 1.3. restore internal keys
78 for k
, v
in internal_keys
.items():
83 # 2. check that this id is not present
84 if "id" in edit_content
:
85 _filter
= self
._get
_project
_filter
(session
)
86 _filter
["id"] = final_content
["id"]
87 _filter
["_id.neq"] = _id
88 if self
.db
.get_one(self
.topic
, _filter
, fail_on_empty
=False):
89 raise EngineException("{} with id '{}' already exists for this project".format(self
.topic
[:-1],
94 def format_on_new(content
, project_id
=None, make_public
=False):
95 BaseTopic
.format_on_new(content
, project_id
=project_id
, make_public
=make_public
)
96 content
["_admin"]["onboardingState"] = "CREATED"
97 content
["_admin"]["operationalState"] = "DISABLED"
98 content
["_admin"]["usageState"] = "NOT_IN_USE"
100 def delete_extra(self
, session
, _id
, db_content
, not_send_msg
=None):
102 Deletes file system storage associated with the descriptor
103 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
104 :param _id: server internal id
105 :param db_content: The database content of the descriptor
106 :param not_send_msg: To not send message (False) or store content (list) instead
107 :return: None if ok or raises EngineException with the problem
109 self
.fs
.file_delete(_id
, ignore_non_exist
=True)
110 self
.fs
.file_delete(_id
+ "_", ignore_non_exist
=True) # remove temp folder
113 def get_one_by_id(db
, session
, topic
, id):
114 # find owned by this project
115 _filter
= BaseTopic
._get
_project
_filter
(session
)
117 desc_list
= db
.get_list(topic
, _filter
)
118 if len(desc_list
) == 1:
120 elif len(desc_list
) > 1:
121 raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic
[:-1], id),
124 # not found any: try to find public
125 _filter
= BaseTopic
._get
_project
_filter
(session
)
127 desc_list
= db
.get_list(topic
, _filter
)
129 raise DbException("Not found any {} with id='{}'".format(topic
[:-1], id), HTTPStatus
.NOT_FOUND
)
130 elif len(desc_list
) == 1:
133 raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
134 topic
[:-1], id), HTTPStatus
.CONFLICT
)
136 def new(self
, rollback
, session
, indata
=None, kwargs
=None, headers
=None):
138 Creates a new almost empty DISABLED entry into database. Due to SOL005, it does not follow normal procedure.
139 Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
140 (self.upload_content)
141 :param rollback: list to append created items at database in case a rollback may to be done
142 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
143 :param indata: data to be inserted
144 :param kwargs: used to override the indata descriptor
145 :param headers: http request headers
146 :return: _id, None: identity of the inserted data; and None as there is not any operation
149 # No needed to capture exceptions
151 self
.check_quota(session
)
155 if "userDefinedData" in indata
:
156 indata
= indata
['userDefinedData']
158 # Override descriptor with query string kwargs
159 self
._update
_input
_with
_kwargs
(indata
, kwargs
)
160 # uncomment when this method is implemented.
161 # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
162 # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
164 content
= {"_admin": {"userDefinedData": indata
}}
165 self
.format_on_new(content
, session
["project_id"], make_public
=session
["public"])
166 _id
= self
.db
.create(self
.topic
, content
)
167 rollback
.append({"topic": self
.topic
, "_id": _id
})
168 self
._send
_msg
("created", {"_id": _id
})
171 def upload_content(self
, session
, _id
, indata
, kwargs
, headers
):
173 Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
174 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
175 :param _id : the nsd,vnfd is already created, this is the id
176 :param indata: http body request
177 :param kwargs: user query string to override parameters. NOT USED
178 :param headers: http request headers
179 :return: True if package is completely uploaded or False if partial content has been uploded
180 Raise exception on error
182 # Check that _id exists and it is valid
183 current_desc
= self
.show(session
, _id
)
185 content_range_text
= headers
.get("Content-Range")
186 expected_md5
= headers
.get("Content-File-MD5")
188 content_type
= headers
.get("Content-Type")
189 if content_type
and "application/gzip" in content_type
or "application/x-gzip" in content_type
or \
190 "application/zip" in content_type
:
192 filename
= headers
.get("Content-Filename")
194 filename
= "package.tar.gz" if compressed
else "package"
195 # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
199 if content_range_text
:
200 content_range
= content_range_text
.replace("-", " ").replace("/", " ").split()
201 if content_range
[0] != "bytes": # TODO check x<y not negative < total....
203 start
= int(content_range
[1])
204 end
= int(content_range
[2]) + 1
205 total
= int(content_range
[3])
208 temp_folder
= _id
+ "_" # all the content is upload here and if ok, it is rename from id_ to is folder
211 if not self
.fs
.file_exists(temp_folder
, 'dir'):
212 raise EngineException("invalid Transaction-Id header", HTTPStatus
.NOT_FOUND
)
214 self
.fs
.file_delete(temp_folder
, ignore_non_exist
=True)
215 self
.fs
.mkdir(temp_folder
)
217 storage
= self
.fs
.get_params()
218 storage
["folder"] = _id
220 file_path
= (temp_folder
, filename
)
221 if self
.fs
.file_exists(file_path
, 'file'):
222 file_size
= self
.fs
.file_size(file_path
)
225 if file_size
!= start
:
226 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
227 file_size
, start
), HTTPStatus
.REQUESTED_RANGE_NOT_SATISFIABLE
)
228 file_pkg
= self
.fs
.file_open(file_path
, 'a+b')
229 if isinstance(indata
, dict):
230 indata_text
= yaml
.safe_dump(indata
, indent
=4, default_flow_style
=False)
231 file_pkg
.write(indata_text
.encode(encoding
="utf-8"))
235 indata_text
= indata
.read(4096)
236 indata_len
+= len(indata_text
)
239 file_pkg
.write(indata_text
)
240 if content_range_text
:
241 if indata_len
!= end
-start
:
242 raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
243 start
, end
-1, indata_len
), HTTPStatus
.REQUESTED_RANGE_NOT_SATISFIABLE
)
245 # TODO update to UPLOADING
252 chunk_data
= file_pkg
.read(1024)
254 file_md5
.update(chunk_data
)
255 chunk_data
= file_pkg
.read(1024)
256 if expected_md5
!= file_md5
.hexdigest():
257 raise EngineException("Error, MD5 mismatch", HTTPStatus
.CONFLICT
)
259 if compressed
== "gzip":
260 tar
= tarfile
.open(mode
='r', fileobj
=file_pkg
)
261 descriptor_file_name
= None
263 tarname
= tarinfo
.name
264 tarname_path
= tarname
.split("/")
265 if not tarname_path
[0] or ".." in tarname_path
: # if start with "/" means absolute path
266 raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
267 if len(tarname_path
) == 1 and not tarinfo
.isdir():
268 raise EngineException("All files must be inside a dir for package descriptor tar.gz")
269 if tarname
.endswith(".yaml") or tarname
.endswith(".json") or tarname
.endswith(".yml"):
270 storage
["pkg-dir"] = tarname_path
[0]
271 if len(tarname_path
) == 2:
272 if descriptor_file_name
:
273 raise EngineException(
274 "Found more than one descriptor file at package descriptor tar.gz")
275 descriptor_file_name
= tarname
276 if not descriptor_file_name
:
277 raise EngineException("Not found any descriptor file at package descriptor tar.gz")
278 storage
["descriptor"] = descriptor_file_name
279 storage
["zipfile"] = filename
280 self
.fs
.file_extract(tar
, temp_folder
)
281 with self
.fs
.file_open((temp_folder
, descriptor_file_name
), "r") as descriptor_file
:
282 content
= descriptor_file
.read()
284 content
= file_pkg
.read()
285 storage
["descriptor"] = descriptor_file_name
= filename
287 if descriptor_file_name
.endswith(".json"):
288 error_text
= "Invalid json format "
289 indata
= json
.load(content
)
291 error_text
= "Invalid yaml format "
292 indata
= yaml
.load(content
, Loader
=yaml
.SafeLoader
)
294 current_desc
["_admin"]["storage"] = storage
295 current_desc
["_admin"]["onboardingState"] = "ONBOARDED"
296 current_desc
["_admin"]["operationalState"] = "ENABLED"
298 indata
= self
._remove
_envelop
(indata
)
300 # Override descriptor with query string kwargs
302 self
._update
_input
_with
_kwargs
(indata
, kwargs
)
303 # it will call overrides method at VnfdTopic or NsdTopic
304 # indata = self._validate_input_edit(indata, force=session["force"])
306 deep_update_rfc7396(current_desc
, indata
)
307 self
.check_conflict_on_edit(session
, current_desc
, indata
, _id
=_id
)
308 current_desc
["_admin"]["modified"] = time()
309 self
.db
.replace(self
.topic
, _id
, current_desc
)
310 self
.fs
.dir_rename(temp_folder
, _id
)
313 self
._send
_msg
("edited", indata
)
315 # TODO if descriptor has changed because kwargs update content and remove cached zip
316 # TODO if zip is not present creates one
319 except EngineException
:
322 raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
323 HTTPStatus
.REQUESTED_RANGE_NOT_SATISFIABLE
)
325 raise EngineException("invalid upload transaction sequence: '{}'".format(e
), HTTPStatus
.BAD_REQUEST
)
326 except tarfile
.ReadError
as e
:
327 raise EngineException("invalid file content {}".format(e
), HTTPStatus
.BAD_REQUEST
)
328 except (ValueError, yaml
.YAMLError
) as e
:
329 raise EngineException(error_text
+ str(e
))
330 except ValidationError
as e
:
331 raise EngineException(e
, HTTPStatus
.UNPROCESSABLE_ENTITY
)
336 def get_file(self
, session
, _id
, path
=None, accept_header
=None):
338 Return the file content of a vnfd or nsd
339 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
340 :param _id: Identity of the vnfd, nsd
341 :param path: artifact path or "$DESCRIPTOR" or None
342 :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
343 :return: opened file plus Accept format or raises an exception
345 accept_text
= accept_zip
= False
347 if 'text/plain' in accept_header
or '*/*' in accept_header
:
349 if 'application/zip' in accept_header
or '*/*' in accept_header
:
350 accept_zip
= 'application/zip'
351 elif 'application/gzip' in accept_header
:
352 accept_zip
= 'application/gzip'
354 if not accept_text
and not accept_zip
:
355 raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
356 http_code
=HTTPStatus
.NOT_ACCEPTABLE
)
358 content
= self
.show(session
, _id
)
359 if content
["_admin"]["onboardingState"] != "ONBOARDED":
360 raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
361 "onboardingState is {}".format(content
["_admin"]["onboardingState"]),
362 http_code
=HTTPStatus
.CONFLICT
)
363 storage
= content
["_admin"]["storage"]
364 if path
is not None and path
!= "$DESCRIPTOR": # artifacts
365 if not storage
.get('pkg-dir'):
366 raise EngineException("Packages does not contains artifacts", http_code
=HTTPStatus
.BAD_REQUEST
)
367 if self
.fs
.file_exists((storage
['folder'], storage
['pkg-dir'], *path
), 'dir'):
368 folder_content
= self
.fs
.dir_ls((storage
['folder'], storage
['pkg-dir'], *path
))
369 return folder_content
, "text/plain"
370 # TODO manage folders in http
372 return self
.fs
.file_open((storage
['folder'], storage
['pkg-dir'], *path
), "rb"),\
373 "application/octet-stream"
375 # pkgtype accept ZIP TEXT -> result
376 # manyfiles yes X -> zip
378 # onefile yes no -> zip
381 if accept_text
and (not storage
.get('pkg-dir') or path
== "$DESCRIPTOR"):
382 return self
.fs
.file_open((storage
['folder'], storage
['descriptor']), "r"), "text/plain"
383 elif storage
.get('pkg-dir') and not accept_zip
:
384 raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
385 "Accept header", http_code
=HTTPStatus
.NOT_ACCEPTABLE
)
387 if not storage
.get('zipfile'):
388 # TODO generate zipfile if not present
389 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
390 "future versions", http_code
=HTTPStatus
.NOT_ACCEPTABLE
)
391 return self
.fs
.file_open((storage
['folder'], storage
['zipfile']), "rb"), accept_zip
393 def pyangbind_validation(self
, item
, data
, force
=False):
397 pybindJSONDecoder
.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data
]}}, None, None, obj
=myvnfd
,
398 path_helper
=True, skip_unknown
=force
)
399 out
= pybindJSON
.dumps(myvnfd
, mode
="ietf")
402 pybindJSONDecoder
.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data
]}}, None, None, obj
=mynsd
,
403 path_helper
=True, skip_unknown
=force
)
404 out
= pybindJSON
.dumps(mynsd
, mode
="ietf")
407 pybindJSONDecoder
.load_ietf_json({'nst': [data
]}, None, None, obj
=mynst
,
408 path_helper
=True, skip_unknown
=force
)
409 out
= pybindJSON
.dumps(mynst
, mode
="ietf")
411 raise EngineException("Not possible to validate '{}' item".format(item
),
412 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
)
414 desc_out
= self
._remove
_envelop
(yaml
.safe_load(out
))
417 except Exception as e
:
418 raise EngineException("Error in pyangbind validation: {}".format(str(e
)),
419 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
422 class VnfdTopic(DescriptorTopic
):
426 def __init__(self
, db
, fs
, msg
, auth
):
427 DescriptorTopic
.__init
__(self
, db
, fs
, msg
, auth
)
430 def _remove_envelop(indata
=None):
433 clean_indata
= indata
434 if clean_indata
.get('vnfd:vnfd-catalog'):
435 clean_indata
= clean_indata
['vnfd:vnfd-catalog']
436 elif clean_indata
.get('vnfd-catalog'):
437 clean_indata
= clean_indata
['vnfd-catalog']
438 if clean_indata
.get('vnfd'):
439 if not isinstance(clean_indata
['vnfd'], list) or len(clean_indata
['vnfd']) != 1:
440 raise EngineException("'vnfd' must be a list of only one element")
441 clean_indata
= clean_indata
['vnfd'][0]
442 elif clean_indata
.get('vnfd:vnfd'):
443 if not isinstance(clean_indata
['vnfd:vnfd'], list) or len(clean_indata
['vnfd:vnfd']) != 1:
444 raise EngineException("'vnfd:vnfd' must be a list of only one element")
445 clean_indata
= clean_indata
['vnfd:vnfd'][0]
448 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
449 super().check_conflict_on_edit(session
, final_content
, edit_content
, _id
)
454 for vdu
in get_iterable(final_content
.get("vdu")):
455 if vdu
.get("pdu-type"):
460 final_content
["_admin"]["type"] = "hnfd" if contains_vdu
else "pnfd"
462 final_content
["_admin"]["type"] = "vnfd"
463 # if neither vud nor pdu do not fill type
465 def check_conflict_on_del(self
, session
, _id
, db_content
):
467 Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
468 that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
470 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
471 :param _id: vnfd internal id
472 :param db_content: The database content of the _id.
473 :return: None or raises EngineException with the conflict
477 descriptor
= db_content
478 descriptor_id
= descriptor
.get("id")
479 if not descriptor_id
: # empty vnfd not uploaded
482 _filter
= self
._get
_project
_filter
(session
)
484 # check vnfrs using this vnfd
485 _filter
["vnfd-id"] = _id
486 if self
.db
.get_list("vnfrs", _filter
):
487 raise EngineException("There is at least one VNF using this descriptor", http_code
=HTTPStatus
.CONFLICT
)
489 # check NSD referencing this VNFD
490 del _filter
["vnfd-id"]
491 _filter
["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
492 if self
.db
.get_list("nsds", _filter
):
493 raise EngineException("There is at least one NSD referencing this descriptor",
494 http_code
=HTTPStatus
.CONFLICT
)
496 def _validate_input_new(self
, indata
, storage_params
, force
=False):
497 indata
= self
.pyangbind_validation("vnfds", indata
, force
)
498 # Cross references validation in the descriptor
499 if indata
.get("vdu"):
500 if not indata
.get("mgmt-interface"):
501 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
502 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
503 if indata
["mgmt-interface"].get("cp"):
504 for cp
in get_iterable(indata
.get("connection-point")):
505 if cp
["name"] == indata
["mgmt-interface"]["cp"]:
508 raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
509 .format(indata
["mgmt-interface"]["cp"]),
510 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
512 for vdu
in get_iterable(indata
.get("vdu")):
515 for interface
in get_iterable(vdu
.get("interface")):
516 if interface
.get("external-connection-point-ref"):
517 if interface
.get("external-connection-point-ref") in ecp_refs
:
518 raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
519 "is referenced by other interface"
520 .format(vdu
["id"], interface
["name"],
521 interface
["external-connection-point-ref"]),
522 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
523 ecp_refs
.append(interface
.get("external-connection-point-ref"))
524 for cp
in get_iterable(indata
.get("connection-point")):
525 if cp
["name"] == interface
["external-connection-point-ref"]:
528 raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
529 "must match an existing connection-point"
530 .format(vdu
["id"], interface
["name"],
531 interface
["external-connection-point-ref"]),
532 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
533 elif interface
.get("internal-connection-point-ref"):
534 if interface
.get("internal-connection-point-ref") in icp_refs
:
535 raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
536 "is referenced by other interface"
537 .format(vdu
["id"], interface
["name"],
538 interface
["internal-connection-point-ref"]),
539 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
540 icp_refs
.append(interface
.get("internal-connection-point-ref"))
541 for internal_cp
in get_iterable(vdu
.get("internal-connection-point")):
542 if interface
["internal-connection-point-ref"] == internal_cp
.get("id"):
545 raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
546 "must match an existing vdu:internal-connection-point"
547 .format(vdu
["id"], interface
["name"],
548 interface
["internal-connection-point-ref"]),
549 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
550 # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
551 if vdu
.get("vdu-configuration"):
552 if vdu
["vdu-configuration"].get("juju"):
553 if not self
._validate
_package
_folders
(storage_params
, 'charms'):
554 raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
555 "package".format(indata
["id"], vdu
["id"]))
556 # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
557 if vdu
.get("cloud-init-file"):
558 if not self
._validate
_package
_folders
(storage_params
, 'cloud_init', vdu
["cloud-init-file"]):
559 raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
560 "package".format(indata
["id"], vdu
["id"]))
561 # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
562 if indata
.get("vnf-configuration"):
563 if indata
["vnf-configuration"].get("juju"):
564 if not self
._validate
_package
_folders
(storage_params
, 'charms'):
565 raise EngineException("Charm defined in vnf[id={}] but not present in "
566 "package".format(indata
["id"]))
567 vld_names
= [] # For detection of duplicated VLD names
568 for ivld
in get_iterable(indata
.get("internal-vld")):
569 # BEGIN Detection of duplicated VLD names
570 ivld_name
= ivld
.get("name")
572 if ivld_name
in vld_names
:
573 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
574 .format(ivld
["name"], indata
["id"], ivld
["id"]),
575 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
577 vld_names
.append(ivld_name
)
578 # END Detection of duplicated VLD names
579 for icp
in get_iterable(ivld
.get("internal-connection-point")):
581 for vdu
in get_iterable(indata
.get("vdu")):
582 for internal_cp
in get_iterable(vdu
.get("internal-connection-point")):
583 if icp
["id-ref"] == internal_cp
["id"]:
589 raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
590 "vdu:internal-connection-point".format(ivld
["id"], icp
["id-ref"]),
591 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
592 if ivld
.get("ip-profile-ref"):
593 for ip_prof
in get_iterable(indata
.get("ip-profiles")):
594 if ip_prof
["name"] == get_iterable(ivld
.get("ip-profile-ref")):
597 raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
598 ivld
["id"], ivld
["ip-profile-ref"]),
599 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
600 for mp
in get_iterable(indata
.get("monitoring-param")):
601 if mp
.get("vdu-monitoring-param"):
603 for vdu
in get_iterable(indata
.get("vdu")):
604 for vmp
in get_iterable(vdu
.get("monitoring-param")):
605 if vmp
["id"] == mp
["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu
["id"] ==\
606 mp
["vdu-monitoring-param"]["vdu-ref"]:
612 raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
613 "defined at vdu[id='{}'] or vdu does not exist"
614 .format(mp
["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
615 mp
["vdu-monitoring-param"]["vdu-ref"]),
616 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
617 elif mp
.get("vdu-metric"):
619 for vdu
in get_iterable(indata
.get("vdu")):
620 if vdu
.get("vdu-configuration"):
621 for metric
in get_iterable(vdu
["vdu-configuration"].get("metrics")):
622 if metric
["name"] == mp
["vdu-metric"]["vdu-metric-name-ref"] and vdu
["id"] == \
623 mp
["vdu-metric"]["vdu-ref"]:
629 raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
630 "vdu[id='{}'] or vdu does not exist"
631 .format(mp
["vdu-metric"]["vdu-metric-name-ref"],
632 mp
["vdu-metric"]["vdu-ref"]),
633 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
635 for sgd
in get_iterable(indata
.get("scaling-group-descriptor")):
636 for sp
in get_iterable(sgd
.get("scaling-policy")):
637 for sc
in get_iterable(sp
.get("scaling-criteria")):
638 for mp
in get_iterable(indata
.get("monitoring-param")):
639 if mp
["id"] == get_iterable(sc
.get("vnf-monitoring-param-ref")):
642 raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
643 "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
644 .format(sgd
["name"], sc
["name"], sc
["vnf-monitoring-param-ref"]),
645 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
646 for sgd_vdu
in get_iterable(sgd
.get("vdu")):
648 for vdu
in get_iterable(indata
.get("vdu")):
649 if vdu
["id"] == sgd_vdu
["vdu-id-ref"]:
655 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
656 .format(sgd
["name"], sgd_vdu
["vdu-id-ref"]),
657 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
658 for sca
in get_iterable(sgd
.get("scaling-config-action")):
659 if not indata
.get("vnf-configuration"):
660 raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
661 "scaling-group-descriptor[name='{}']:scaling-config-action"
662 .format(sgd
["name"]),
663 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
664 for primitive
in get_iterable(indata
["vnf-configuration"].get("config-primitive")):
665 if primitive
["name"] == sca
["vnf-config-primitive-name-ref"]:
668 raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
669 "primitive-name-ref='{}' does not match any "
670 "vnf-configuration:config-primitive:name"
671 .format(sgd
["name"], sca
["vnf-config-primitive-name-ref"]),
672 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
675 def _validate_input_edit(self
, indata
, force
=False):
676 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
679 def _validate_package_folders(self
, storage_params
, folder
, file=None):
680 if not storage_params
or not storage_params
.get("pkg-dir"):
683 if self
.fs
.file_exists("{}_".format(storage_params
["folder"]), 'dir'):
684 f
= "{}_/{}/{}".format(storage_params
["folder"], storage_params
["pkg-dir"], folder
)
686 f
= "{}/{}/{}".format(storage_params
["folder"], storage_params
["pkg-dir"], folder
)
688 return self
.fs
.file_exists("{}/{}".format(f
, file), 'file')
690 if self
.fs
.file_exists(f
, 'dir'):
691 if self
.fs
.dir_ls(f
):
695 def delete_extra(self
, session
, _id
, db_content
, not_send_msg
=None):
697 Deletes associate file system storage (via super)
698 Deletes associated vnfpkgops from database.
699 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
700 :param _id: server internal id
701 :param db_content: The database content of the descriptor
703 :raises: FsException in case of error while deleting associated storage
705 super().delete_extra(session
, _id
, db_content
, not_send_msg
)
706 self
.db
.del_list("vnfpkgops", {"vnfPkgId": _id
})
709 class NsdTopic(DescriptorTopic
):
713 def __init__(self
, db
, fs
, msg
, auth
):
714 DescriptorTopic
.__init
__(self
, db
, fs
, msg
, auth
)
717 def _remove_envelop(indata
=None):
720 clean_indata
= indata
722 if clean_indata
.get('nsd:nsd-catalog'):
723 clean_indata
= clean_indata
['nsd:nsd-catalog']
724 elif clean_indata
.get('nsd-catalog'):
725 clean_indata
= clean_indata
['nsd-catalog']
726 if clean_indata
.get('nsd'):
727 if not isinstance(clean_indata
['nsd'], list) or len(clean_indata
['nsd']) != 1:
728 raise EngineException("'nsd' must be a list of only one element")
729 clean_indata
= clean_indata
['nsd'][0]
730 elif clean_indata
.get('nsd:nsd'):
731 if not isinstance(clean_indata
['nsd:nsd'], list) or len(clean_indata
['nsd:nsd']) != 1:
732 raise EngineException("'nsd:nsd' must be a list of only one element")
733 clean_indata
= clean_indata
['nsd:nsd'][0]
736 def _validate_input_new(self
, indata
, storage_params
, force
=False):
737 indata
= self
.pyangbind_validation("nsds", indata
, force
)
738 # Cross references validation in the descriptor
739 # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
740 for vld
in get_iterable(indata
.get("vld")):
741 if vld
.get("mgmt-network") and vld
.get("ip-profile-ref"):
742 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
743 " You cannot set an ip-profile when mgmt-network is True"
744 .format(vld
["id"]), http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
745 for vnfd_cp
in get_iterable(vld
.get("vnfd-connection-point-ref")):
746 for constituent_vnfd
in get_iterable(indata
.get("constituent-vnfd")):
747 if vnfd_cp
["member-vnf-index-ref"] == constituent_vnfd
["member-vnf-index"]:
748 if vnfd_cp
.get("vnfd-id-ref") and vnfd_cp
["vnfd-id-ref"] != constituent_vnfd
["vnfd-id-ref"]:
749 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
750 "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
751 " '{}'".format(vld
["id"], vnfd_cp
["vnfd-id-ref"],
752 constituent_vnfd
["member-vnf-index"],
753 constituent_vnfd
["vnfd-id-ref"]),
754 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
757 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
758 "does not match any constituent-vnfd:member-vnf-index"
759 .format(vld
["id"], vnfd_cp
["member-vnf-index-ref"]),
760 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
762 for fgd
in get_iterable(indata
.get("vnffgd")):
763 for cls
in get_iterable(fgd
.get("classifier")):
764 rspref
= cls
.get("rsp-id-ref")
765 for rsp
in get_iterable(fgd
.get("rsp")):
766 rspid
= rsp
.get("id")
767 if rspid
and rspref
and rspid
== rspref
:
770 raise EngineException(
771 "Error at vnffgd[id='{}']:classifier[id='{}']:rsp-id-ref '{}' does not match any rsp:id"
772 .format(fgd
["id"], cls
["id"], rspref
),
773 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
776 def _validate_input_edit(self
, indata
, force
=False):
777 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
780 def _check_descriptor_dependencies(self
, session
, descriptor
):
782 Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
783 connection points are ok
784 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
785 :param descriptor: descriptor to be inserted or edit
786 :return: None or raises exception
790 member_vnfd_index
= {}
791 if descriptor
.get("constituent-vnfd") and not session
["force"]:
792 for vnf
in descriptor
["constituent-vnfd"]:
793 vnfd_id
= vnf
["vnfd-id-ref"]
794 filter_q
= self
._get
_project
_filter
(session
)
795 filter_q
["id"] = vnfd_id
796 vnf_list
= self
.db
.get_list("vnfds", filter_q
)
798 raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
799 "existing vnfd".format(vnfd_id
), http_code
=HTTPStatus
.CONFLICT
)
800 # elif len(vnf_list) > 1:
801 # raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
802 # http_code=HTTPStatus.CONFLICT)
803 member_vnfd_index
[vnf
["member-vnf-index"]] = vnf_list
[0]
805 # Cross references validation in the descriptor and vnfd connection point validation
806 for vld
in get_iterable(descriptor
.get("vld")):
807 for referenced_vnfd_cp
in get_iterable(vld
.get("vnfd-connection-point-ref")):
808 # look if this vnfd contains this connection point
809 vnfd
= member_vnfd_index
.get(referenced_vnfd_cp
["member-vnf-index-ref"])
810 for vnfd_cp
in get_iterable(vnfd
.get("connection-point")):
811 if referenced_vnfd_cp
.get("vnfd-connection-point-ref") == vnfd_cp
["name"]:
814 raise EngineException(
815 "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
816 "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
817 .format(vld
["id"], referenced_vnfd_cp
["member-vnf-index-ref"],
818 referenced_vnfd_cp
["vnfd-connection-point-ref"], vnfd
["id"]),
819 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
821 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
822 super().check_conflict_on_edit(session
, final_content
, edit_content
, _id
)
824 self
._check
_descriptor
_dependencies
(session
, final_content
)
826 def check_conflict_on_del(self
, session
, _id
, db_content
):
828 Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
829 that NSD can be public and be used by other projects.
830 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
831 :param _id: nsd internal id
832 :param db_content: The database content of the _id
833 :return: None or raises EngineException with the conflict
837 descriptor
= db_content
838 descriptor_id
= descriptor
.get("id")
839 if not descriptor_id
: # empty nsd not uploaded
842 # check NSD used by NS
843 _filter
= self
._get
_project
_filter
(session
)
844 _filter
["nsd-id"] = _id
845 if self
.db
.get_list("nsrs", _filter
):
846 raise EngineException("There is at least one NS using this descriptor", http_code
=HTTPStatus
.CONFLICT
)
848 # check NSD referenced by NST
849 del _filter
["nsd-id"]
850 _filter
["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
851 if self
.db
.get_list("nsts", _filter
):
852 raise EngineException("There is at least one NetSlice Template referencing this descriptor",
853 http_code
=HTTPStatus
.CONFLICT
)
856 class NstTopic(DescriptorTopic
):
859 quota_name
= "slice_templates"
861 def __init__(self
, db
, fs
, msg
, auth
):
862 DescriptorTopic
.__init
__(self
, db
, fs
, msg
, auth
)
865 def _remove_envelop(indata
=None):
868 clean_indata
= indata
870 if clean_indata
.get('nst'):
871 if not isinstance(clean_indata
['nst'], list) or len(clean_indata
['nst']) != 1:
872 raise EngineException("'nst' must be a list only one element")
873 clean_indata
= clean_indata
['nst'][0]
874 elif clean_indata
.get('nst:nst'):
875 if not isinstance(clean_indata
['nst:nst'], list) or len(clean_indata
['nst:nst']) != 1:
876 raise EngineException("'nst:nst' must be a list only one element")
877 clean_indata
= clean_indata
['nst:nst'][0]
880 def _validate_input_edit(self
, indata
, force
=False):
881 # TODO validate with pyangbind, serialize
884 def _validate_input_new(self
, indata
, storage_params
, force
=False):
885 indata
= self
.pyangbind_validation("nsts", indata
, force
)
888 def _check_descriptor_dependencies(self
, session
, descriptor
):
890 Check that the dependent descriptors exist on a new descriptor or edition
891 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
892 :param descriptor: descriptor to be inserted or edit
893 :return: None or raises exception
895 if not descriptor
.get("netslice-subnet"):
897 for nsd
in descriptor
["netslice-subnet"]:
898 nsd_id
= nsd
["nsd-ref"]
899 filter_q
= self
._get
_project
_filter
(session
)
900 filter_q
["id"] = nsd_id
901 if not self
.db
.get_list("nsds", filter_q
):
902 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
903 "existing nsd".format(nsd_id
), http_code
=HTTPStatus
.CONFLICT
)
905 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
906 super().check_conflict_on_edit(session
, final_content
, edit_content
, _id
)
908 self
._check
_descriptor
_dependencies
(session
, final_content
)
910 def check_conflict_on_del(self
, session
, _id
, db_content
):
912 Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
913 that NST can be public and be used by other projects.
914 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
915 :param _id: nst internal id
916 :param db_content: The database content of the _id.
917 :return: None or raises EngineException with the conflict
919 # TODO: Check this method
922 # Get Network Slice Template from Database
923 _filter
= self
._get
_project
_filter
(session
)
924 _filter
["_admin.nst-id"] = _id
925 if self
.db
.get_list("nsis", _filter
):
926 raise EngineException("there is at least one Netslice Instance using this descriptor",
927 http_code
=HTTPStatus
.CONFLICT
)
930 class PduTopic(BaseTopic
):
934 schema_new
= pdu_new_schema
935 schema_edit
= pdu_edit_schema
937 def __init__(self
, db
, fs
, msg
, auth
):
938 BaseTopic
.__init
__(self
, db
, fs
, msg
, auth
)
941 def format_on_new(content
, project_id
=None, make_public
=False):
942 BaseTopic
.format_on_new(content
, project_id
=project_id
, make_public
=make_public
)
943 content
["_admin"]["onboardingState"] = "CREATED"
944 content
["_admin"]["operationalState"] = "ENABLED"
945 content
["_admin"]["usageState"] = "NOT_IN_USE"
947 def check_conflict_on_del(self
, session
, _id
, db_content
):
949 Check that there is not any vnfr that uses this PDU
950 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
951 :param _id: pdu internal id
952 :param db_content: The database content of the _id.
953 :return: None or raises EngineException with the conflict
958 _filter
= self
._get
_project
_filter
(session
)
959 _filter
["vdur.pdu-id"] = _id
960 if self
.db
.get_list("vnfrs", _filter
):
961 raise EngineException("There is at least one VNF using this PDU", http_code
=HTTPStatus
.CONFLICT
)
964 class VnfPkgOpTopic(BaseTopic
):
967 schema_new
= vnfpkgop_new_schema
970 def __init__(self
, db
, fs
, msg
, auth
):
971 BaseTopic
.__init
__(self
, db
, fs
, msg
, auth
)
973 def edit(self
, session
, _id
, indata
=None, kwargs
=None, content
=None):
974 raise EngineException("Method 'edit' not allowed for topic '{}'".format(self
.topic
),
975 HTTPStatus
.METHOD_NOT_ALLOWED
)
977 def delete(self
, session
, _id
, dry_run
=False):
978 raise EngineException("Method 'delete' not allowed for topic '{}'".format(self
.topic
),
979 HTTPStatus
.METHOD_NOT_ALLOWED
)
981 def delete_list(self
, session
, filter_q
=None):
982 raise EngineException("Method 'delete_list' not allowed for topic '{}'".format(self
.topic
),
983 HTTPStatus
.METHOD_NOT_ALLOWED
)
985 def new(self
, rollback
, session
, indata
=None, kwargs
=None, headers
=None):
987 Creates a new entry into database.
988 :param rollback: list to append created items at database in case a rollback may to be done
989 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
990 :param indata: data to be inserted
991 :param kwargs: used to override the indata descriptor
992 :param headers: http request headers
994 _id: identity of the inserted data.
997 self
._update
_input
_with
_kwargs
(indata
, kwargs
)
998 validate_input(indata
, self
.schema_new
)
999 vnfpkg_id
= indata
["vnfPkgId"]
1000 filter_q
= BaseTopic
._get
_project
_filter
(session
)
1001 filter_q
["_id"] = vnfpkg_id
1002 vnfd
= self
.db
.get_one("vnfds", filter_q
)
1003 operation
= indata
["lcmOperationType"]
1004 kdu_name
= indata
["kdu_name"]
1005 for kdu
in vnfd
.get("kdu", []):
1006 if kdu
["name"] == kdu_name
:
1007 helm_chart
= kdu
.get("helm-chart")
1008 juju_bundle
= kdu
.get("juju-bundle")
1011 raise EngineException("Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id
, kdu_name
))
1013 indata
["helm-chart"] = helm_chart
1014 match
= fullmatch(r
"([^/]*)/([^/]*)", helm_chart
)
1015 repo_name
= match
.group(1) if match
else None
1017 indata
["juju-bundle"] = juju_bundle
1018 match
= fullmatch(r
"([^/]*)/([^/]*)", juju_bundle
)
1019 repo_name
= match
.group(1) if match
else None
1021 raise EngineException("Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']"
1022 .format(vnfpkg_id
, kdu_name
))
1025 filter_q
["name"] = repo_name
1026 repo
= self
.db
.get_one("k8srepos", filter_q
)
1027 k8srepo_id
= repo
.get("_id")
1028 k8srepo_url
= repo
.get("url")
1032 indata
["k8srepoId"] = k8srepo_id
1033 indata
["k8srepo_url"] = k8srepo_url
1034 vnfpkgop_id
= str(uuid4())
1037 "operationState": "PROCESSING",
1038 "vnfPkgId": vnfpkg_id
,
1039 "lcmOperationType": operation
,
1040 "isAutomaticInvocation": False,
1041 "isCancelPending": False,
1042 "operationParams": indata
,
1044 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id
,
1045 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id
,
1048 self
.format_on_new(vnfpkgop_desc
, session
["project_id"], make_public
=session
["public"])
1049 ctime
= vnfpkgop_desc
["_admin"]["created"]
1050 vnfpkgop_desc
["statusEnteredTime"] = ctime
1051 vnfpkgop_desc
["startTime"] = ctime
1052 self
.db
.create(self
.topic
, vnfpkgop_desc
)
1053 rollback
.append({"topic": self
.topic
, "_id": vnfpkgop_id
})
1054 self
.msg
.write(self
.topic_msg
, operation
, vnfpkgop_desc
)
1055 return vnfpkgop_id
, None