1 # -*- coding: utf-8 -*-
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
20 from hashlib
import md5
21 from osm_common
.dbbase
import DbException
, deep_update_rfc7396
22 from http
import HTTPStatus
24 from uuid
import uuid4
25 from re
import fullmatch
26 from osm_nbi
.validation
import ValidationError
, pdu_new_schema
, pdu_edit_schema
, \
27 validate_input
, vnfpkgop_new_schema
28 from osm_nbi
.base_topic
import BaseTopic
, EngineException
, get_iterable
29 from osm_im
.vnfd
import vnfd
as vnfd_im
30 from osm_im
.nsd
import nsd
as nsd_im
31 from osm_im
.nst
import nst
as nst_im
32 from pyangbind
.lib
.serialise
import pybindJSONDecoder
33 import pyangbind
.lib
.pybindJSON
as pybindJSON
35 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
38 class DescriptorTopic(BaseTopic
):
40 def __init__(self
, db
, fs
, msg
, auth
):
41 BaseTopic
.__init
__(self
, db
, fs
, msg
, auth
)
43 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
44 super().check_conflict_on_edit(session
, final_content
, edit_content
, _id
)
46 def _check_unique_id_name(descriptor
, position
=""):
47 for desc_key
, desc_item
in descriptor
.items():
48 if isinstance(desc_item
, list) and desc_item
:
51 for index
, list_item
in enumerate(desc_item
):
52 if isinstance(list_item
, dict):
53 _check_unique_id_name(list_item
, "{}.{}[{}]"
54 .format(position
, desc_key
, index
))
56 if index
== 0 and (list_item
.get("id") or list_item
.get("name")):
57 desc_item_id
= "id" if list_item
.get("id") else "name"
58 if desc_item_id
and list_item
.get(desc_item_id
):
59 if list_item
[desc_item_id
] in used_ids
:
60 position
= "{}.{}[{}]".format(position
, desc_key
, index
)
61 raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
62 .format(desc_item_id
, list_item
[desc_item_id
],
63 position
), HTTPStatus
.UNPROCESSABLE_ENTITY
)
64 used_ids
.append(list_item
[desc_item_id
])
65 _check_unique_id_name(final_content
)
66 # 1. validate again with pyangbind
67 # 1.1. remove internal keys
69 for k
in ("_id", "_admin"):
70 if k
in final_content
:
71 internal_keys
[k
] = final_content
.pop(k
)
72 storage_params
= internal_keys
["_admin"].get("storage")
73 serialized
= self
._validate
_input
_new
(final_content
, storage_params
, session
["force"])
74 # 1.2. modify final_content with a serialized version
76 final_content
.update(serialized
)
77 # 1.3. restore internal keys
78 for k
, v
in internal_keys
.items():
83 # 2. check that this id is not present
84 if "id" in edit_content
:
85 _filter
= self
._get
_project
_filter
(session
)
86 _filter
["id"] = final_content
["id"]
87 _filter
["_id.neq"] = _id
88 if self
.db
.get_one(self
.topic
, _filter
, fail_on_empty
=False):
89 raise EngineException("{} with id '{}' already exists for this project".format(self
.topic
[:-1],
94 def format_on_new(content
, project_id
=None, make_public
=False):
95 BaseTopic
.format_on_new(content
, project_id
=project_id
, make_public
=make_public
)
96 content
["_admin"]["onboardingState"] = "CREATED"
97 content
["_admin"]["operationalState"] = "DISABLED"
98 content
["_admin"]["usageState"] = "NOT_IN_USE"
100 def delete_extra(self
, session
, _id
, db_content
, not_send_msg
=None):
102 Deletes file system storage associated with the descriptor
103 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
104 :param _id: server internal id
105 :param db_content: The database content of the descriptor
106 :param not_send_msg: To not send message (False) or store content (list) instead
107 :return: None if ok or raises EngineException with the problem
109 self
.fs
.file_delete(_id
, ignore_non_exist
=True)
110 self
.fs
.file_delete(_id
+ "_", ignore_non_exist
=True) # remove temp folder
113 def get_one_by_id(db
, session
, topic
, id):
114 # find owned by this project
115 _filter
= BaseTopic
._get
_project
_filter
(session
)
117 desc_list
= db
.get_list(topic
, _filter
)
118 if len(desc_list
) == 1:
120 elif len(desc_list
) > 1:
121 raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic
[:-1], id),
124 # not found any: try to find public
125 _filter
= BaseTopic
._get
_project
_filter
(session
)
127 desc_list
= db
.get_list(topic
, _filter
)
129 raise DbException("Not found any {} with id='{}'".format(topic
[:-1], id), HTTPStatus
.NOT_FOUND
)
130 elif len(desc_list
) == 1:
133 raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
134 topic
[:-1], id), HTTPStatus
.CONFLICT
)
136 def new(self
, rollback
, session
, indata
=None, kwargs
=None, headers
=None):
138 Creates a new almost empty DISABLED entry into database. Due to SOL005, it does not follow normal procedure.
139 Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
140 (self.upload_content)
141 :param rollback: list to append created items at database in case a rollback may to be done
142 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
143 :param indata: data to be inserted
144 :param kwargs: used to override the indata descriptor
145 :param headers: http request headers
146 :return: _id, None: identity of the inserted data; and None as there is not any operation
151 self
.check_quota(session
)
155 if "userDefinedData" in indata
:
156 indata
= indata
['userDefinedData']
158 # Override descriptor with query string kwargs
159 self
._update
_input
_with
_kwargs
(indata
, kwargs
)
160 # uncomment when this method is implemented.
161 # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
162 # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
164 content
= {"_admin": {"userDefinedData": indata
}}
165 self
.format_on_new(content
, session
["project_id"], make_public
=session
["public"])
166 _id
= self
.db
.create(self
.topic
, content
)
167 rollback
.append({"topic": self
.topic
, "_id": _id
})
168 self
._send
_msg
("created", {"_id": _id
})
170 except ValidationError
as e
:
171 raise EngineException(e
, HTTPStatus
.UNPROCESSABLE_ENTITY
)
173 def upload_content(self
, session
, _id
, indata
, kwargs
, headers
):
175 Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
176 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
177 :param _id : the nsd,vnfd is already created, this is the id
178 :param indata: http body request
179 :param kwargs: user query string to override parameters. NOT USED
180 :param headers: http request headers
181 :return: True if package is completely uploaded or False if partial content has been uploded
182 Raise exception on error
184 # Check that _id exists and it is valid
185 current_desc
= self
.show(session
, _id
)
187 content_range_text
= headers
.get("Content-Range")
188 expected_md5
= headers
.get("Content-File-MD5")
190 content_type
= headers
.get("Content-Type")
191 if content_type
and "application/gzip" in content_type
or "application/x-gzip" in content_type
or \
192 "application/zip" in content_type
:
194 filename
= headers
.get("Content-Filename")
196 filename
= "package.tar.gz" if compressed
else "package"
197 # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
201 if content_range_text
:
202 content_range
= content_range_text
.replace("-", " ").replace("/", " ").split()
203 if content_range
[0] != "bytes": # TODO check x<y not negative < total....
205 start
= int(content_range
[1])
206 end
= int(content_range
[2]) + 1
207 total
= int(content_range
[3])
210 temp_folder
= _id
+ "_" # all the content is upload here and if ok, it is rename from id_ to is folder
213 if not self
.fs
.file_exists(temp_folder
, 'dir'):
214 raise EngineException("invalid Transaction-Id header", HTTPStatus
.NOT_FOUND
)
216 self
.fs
.file_delete(temp_folder
, ignore_non_exist
=True)
217 self
.fs
.mkdir(temp_folder
)
219 storage
= self
.fs
.get_params()
220 storage
["folder"] = _id
222 file_path
= (temp_folder
, filename
)
223 if self
.fs
.file_exists(file_path
, 'file'):
224 file_size
= self
.fs
.file_size(file_path
)
227 if file_size
!= start
:
228 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
229 file_size
, start
), HTTPStatus
.REQUESTED_RANGE_NOT_SATISFIABLE
)
230 file_pkg
= self
.fs
.file_open(file_path
, 'a+b')
231 if isinstance(indata
, dict):
232 indata_text
= yaml
.safe_dump(indata
, indent
=4, default_flow_style
=False)
233 file_pkg
.write(indata_text
.encode(encoding
="utf-8"))
237 indata_text
= indata
.read(4096)
238 indata_len
+= len(indata_text
)
241 file_pkg
.write(indata_text
)
242 if content_range_text
:
243 if indata_len
!= end
-start
:
244 raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
245 start
, end
-1, indata_len
), HTTPStatus
.REQUESTED_RANGE_NOT_SATISFIABLE
)
247 # TODO update to UPLOADING
254 chunk_data
= file_pkg
.read(1024)
256 file_md5
.update(chunk_data
)
257 chunk_data
= file_pkg
.read(1024)
258 if expected_md5
!= file_md5
.hexdigest():
259 raise EngineException("Error, MD5 mismatch", HTTPStatus
.CONFLICT
)
261 if compressed
== "gzip":
262 tar
= tarfile
.open(mode
='r', fileobj
=file_pkg
)
263 descriptor_file_name
= None
265 tarname
= tarinfo
.name
266 tarname_path
= tarname
.split("/")
267 if not tarname_path
[0] or ".." in tarname_path
: # if start with "/" means absolute path
268 raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
269 if len(tarname_path
) == 1 and not tarinfo
.isdir():
270 raise EngineException("All files must be inside a dir for package descriptor tar.gz")
271 if tarname
.endswith(".yaml") or tarname
.endswith(".json") or tarname
.endswith(".yml"):
272 storage
["pkg-dir"] = tarname_path
[0]
273 if len(tarname_path
) == 2:
274 if descriptor_file_name
:
275 raise EngineException(
276 "Found more than one descriptor file at package descriptor tar.gz")
277 descriptor_file_name
= tarname
278 if not descriptor_file_name
:
279 raise EngineException("Not found any descriptor file at package descriptor tar.gz")
280 storage
["descriptor"] = descriptor_file_name
281 storage
["zipfile"] = filename
282 self
.fs
.file_extract(tar
, temp_folder
)
283 with self
.fs
.file_open((temp_folder
, descriptor_file_name
), "r") as descriptor_file
:
284 content
= descriptor_file
.read()
286 content
= file_pkg
.read()
287 storage
["descriptor"] = descriptor_file_name
= filename
289 if descriptor_file_name
.endswith(".json"):
290 error_text
= "Invalid json format "
291 indata
= json
.load(content
)
293 error_text
= "Invalid yaml format "
294 indata
= yaml
.load(content
, Loader
=yaml
.SafeLoader
)
296 current_desc
["_admin"]["storage"] = storage
297 current_desc
["_admin"]["onboardingState"] = "ONBOARDED"
298 current_desc
["_admin"]["operationalState"] = "ENABLED"
300 indata
= self
._remove
_envelop
(indata
)
302 # Override descriptor with query string kwargs
304 self
._update
_input
_with
_kwargs
(indata
, kwargs
)
305 # it will call overrides method at VnfdTopic or NsdTopic
306 # indata = self._validate_input_edit(indata, force=session["force"])
308 deep_update_rfc7396(current_desc
, indata
)
309 self
.check_conflict_on_edit(session
, current_desc
, indata
, _id
=_id
)
310 current_desc
["_admin"]["modified"] = time()
311 self
.db
.replace(self
.topic
, _id
, current_desc
)
312 self
.fs
.dir_rename(temp_folder
, _id
)
315 self
._send
_msg
("edited", indata
)
317 # TODO if descriptor has changed because kwargs update content and remove cached zip
318 # TODO if zip is not present creates one
321 except EngineException
:
324 raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
325 HTTPStatus
.REQUESTED_RANGE_NOT_SATISFIABLE
)
327 raise EngineException("invalid upload transaction sequence: '{}'".format(e
), HTTPStatus
.BAD_REQUEST
)
328 except tarfile
.ReadError
as e
:
329 raise EngineException("invalid file content {}".format(e
), HTTPStatus
.BAD_REQUEST
)
330 except (ValueError, yaml
.YAMLError
) as e
:
331 raise EngineException(error_text
+ str(e
))
332 except ValidationError
as e
:
333 raise EngineException(e
, HTTPStatus
.UNPROCESSABLE_ENTITY
)
338 def get_file(self
, session
, _id
, path
=None, accept_header
=None):
340 Return the file content of a vnfd or nsd
341 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
342 :param _id: Identity of the vnfd, nsd
343 :param path: artifact path or "$DESCRIPTOR" or None
344 :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
345 :return: opened file plus Accept format or raises an exception
347 accept_text
= accept_zip
= False
349 if 'text/plain' in accept_header
or '*/*' in accept_header
:
351 if 'application/zip' in accept_header
or '*/*' in accept_header
:
352 accept_zip
= 'application/zip'
353 elif 'application/gzip' in accept_header
:
354 accept_zip
= 'application/gzip'
356 if not accept_text
and not accept_zip
:
357 raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
358 http_code
=HTTPStatus
.NOT_ACCEPTABLE
)
360 content
= self
.show(session
, _id
)
361 if content
["_admin"]["onboardingState"] != "ONBOARDED":
362 raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
363 "onboardingState is {}".format(content
["_admin"]["onboardingState"]),
364 http_code
=HTTPStatus
.CONFLICT
)
365 storage
= content
["_admin"]["storage"]
366 if path
is not None and path
!= "$DESCRIPTOR": # artifacts
367 if not storage
.get('pkg-dir'):
368 raise EngineException("Packages does not contains artifacts", http_code
=HTTPStatus
.BAD_REQUEST
)
369 if self
.fs
.file_exists((storage
['folder'], storage
['pkg-dir'], *path
), 'dir'):
370 folder_content
= self
.fs
.dir_ls((storage
['folder'], storage
['pkg-dir'], *path
))
371 return folder_content
, "text/plain"
372 # TODO manage folders in http
374 return self
.fs
.file_open((storage
['folder'], storage
['pkg-dir'], *path
), "rb"),\
375 "application/octet-stream"
377 # pkgtype accept ZIP TEXT -> result
378 # manyfiles yes X -> zip
380 # onefile yes no -> zip
383 if accept_text
and (not storage
.get('pkg-dir') or path
== "$DESCRIPTOR"):
384 return self
.fs
.file_open((storage
['folder'], storage
['descriptor']), "r"), "text/plain"
385 elif storage
.get('pkg-dir') and not accept_zip
:
386 raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
387 "Accept header", http_code
=HTTPStatus
.NOT_ACCEPTABLE
)
389 if not storage
.get('zipfile'):
390 # TODO generate zipfile if not present
391 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
392 "future versions", http_code
=HTTPStatus
.NOT_ACCEPTABLE
)
393 return self
.fs
.file_open((storage
['folder'], storage
['zipfile']), "rb"), accept_zip
395 def pyangbind_validation(self
, item
, data
, force
=False):
399 pybindJSONDecoder
.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data
]}}, None, None, obj
=myvnfd
,
400 path_helper
=True, skip_unknown
=force
)
401 out
= pybindJSON
.dumps(myvnfd
, mode
="ietf")
404 pybindJSONDecoder
.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data
]}}, None, None, obj
=mynsd
,
405 path_helper
=True, skip_unknown
=force
)
406 out
= pybindJSON
.dumps(mynsd
, mode
="ietf")
409 pybindJSONDecoder
.load_ietf_json({'nst': [data
]}, None, None, obj
=mynst
,
410 path_helper
=True, skip_unknown
=force
)
411 out
= pybindJSON
.dumps(mynst
, mode
="ietf")
413 raise EngineException("Not possible to validate '{}' item".format(item
),
414 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
)
416 desc_out
= self
._remove
_envelop
(yaml
.safe_load(out
))
419 except Exception as e
:
420 raise EngineException("Error in pyangbind validation: {}".format(str(e
)),
421 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
424 class VnfdTopic(DescriptorTopic
):
428 def __init__(self
, db
, fs
, msg
, auth
):
429 DescriptorTopic
.__init
__(self
, db
, fs
, msg
, auth
)
432 def _remove_envelop(indata
=None):
435 clean_indata
= indata
436 if clean_indata
.get('vnfd:vnfd-catalog'):
437 clean_indata
= clean_indata
['vnfd:vnfd-catalog']
438 elif clean_indata
.get('vnfd-catalog'):
439 clean_indata
= clean_indata
['vnfd-catalog']
440 if clean_indata
.get('vnfd'):
441 if not isinstance(clean_indata
['vnfd'], list) or len(clean_indata
['vnfd']) != 1:
442 raise EngineException("'vnfd' must be a list of only one element")
443 clean_indata
= clean_indata
['vnfd'][0]
444 elif clean_indata
.get('vnfd:vnfd'):
445 if not isinstance(clean_indata
['vnfd:vnfd'], list) or len(clean_indata
['vnfd:vnfd']) != 1:
446 raise EngineException("'vnfd:vnfd' must be a list of only one element")
447 clean_indata
= clean_indata
['vnfd:vnfd'][0]
450 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
451 super().check_conflict_on_edit(session
, final_content
, edit_content
, _id
)
456 for vdu
in get_iterable(final_content
.get("vdu")):
457 if vdu
.get("pdu-type"):
462 final_content
["_admin"]["type"] = "hnfd" if contains_vdu
else "pnfd"
464 final_content
["_admin"]["type"] = "vnfd"
465 # if neither vud nor pdu do not fill type
467 def check_conflict_on_del(self
, session
, _id
, db_content
):
469 Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
470 that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
472 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
473 :param _id: vnfd internal id
474 :param db_content: The database content of the _id.
475 :return: None or raises EngineException with the conflict
479 descriptor
= db_content
480 descriptor_id
= descriptor
.get("id")
481 if not descriptor_id
: # empty vnfd not uploaded
484 _filter
= self
._get
_project
_filter
(session
)
486 # check vnfrs using this vnfd
487 _filter
["vnfd-id"] = _id
488 if self
.db
.get_list("vnfrs", _filter
):
489 raise EngineException("There is at least one VNF using this descriptor", http_code
=HTTPStatus
.CONFLICT
)
491 # check NSD referencing this VNFD
492 del _filter
["vnfd-id"]
493 _filter
["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
494 if self
.db
.get_list("nsds", _filter
):
495 raise EngineException("There is at least one NSD referencing this descriptor",
496 http_code
=HTTPStatus
.CONFLICT
)
498 def _validate_input_new(self
, indata
, storage_params
, force
=False):
499 indata
= self
.pyangbind_validation("vnfds", indata
, force
)
500 # Cross references validation in the descriptor
501 if indata
.get("vdu"):
502 if not indata
.get("mgmt-interface"):
503 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
504 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
505 if indata
["mgmt-interface"].get("cp"):
506 for cp
in get_iterable(indata
.get("connection-point")):
507 if cp
["name"] == indata
["mgmt-interface"]["cp"]:
510 raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
511 .format(indata
["mgmt-interface"]["cp"]),
512 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
514 for vdu
in get_iterable(indata
.get("vdu")):
517 for interface
in get_iterable(vdu
.get("interface")):
518 if interface
.get("external-connection-point-ref"):
519 if interface
.get("external-connection-point-ref") in ecp_refs
:
520 raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
521 "is referenced by other interface"
522 .format(vdu
["id"], interface
["name"],
523 interface
["external-connection-point-ref"]),
524 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
525 ecp_refs
.append(interface
.get("external-connection-point-ref"))
526 for cp
in get_iterable(indata
.get("connection-point")):
527 if cp
["name"] == interface
["external-connection-point-ref"]:
530 raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
531 "must match an existing connection-point"
532 .format(vdu
["id"], interface
["name"],
533 interface
["external-connection-point-ref"]),
534 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
535 elif interface
.get("internal-connection-point-ref"):
536 if interface
.get("internal-connection-point-ref") in icp_refs
:
537 raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
538 "is referenced by other interface"
539 .format(vdu
["id"], interface
["name"],
540 interface
["internal-connection-point-ref"]),
541 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
542 icp_refs
.append(interface
.get("internal-connection-point-ref"))
543 for internal_cp
in get_iterable(vdu
.get("internal-connection-point")):
544 if interface
["internal-connection-point-ref"] == internal_cp
.get("id"):
547 raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
548 "must match an existing vdu:internal-connection-point"
549 .format(vdu
["id"], interface
["name"],
550 interface
["internal-connection-point-ref"]),
551 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
552 # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
553 if vdu
.get("vdu-configuration"):
554 if vdu
["vdu-configuration"].get("juju"):
555 if not self
._validate
_package
_folders
(storage_params
, 'charms'):
556 raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
557 "package".format(indata
["id"], vdu
["id"]))
558 # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
559 if vdu
.get("cloud-init-file"):
560 if not self
._validate
_package
_folders
(storage_params
, 'cloud_init', vdu
["cloud-init-file"]):
561 raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
562 "package".format(indata
["id"], vdu
["id"]))
563 # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
564 if indata
.get("vnf-configuration"):
565 if indata
["vnf-configuration"].get("juju"):
566 if not self
._validate
_package
_folders
(storage_params
, 'charms'):
567 raise EngineException("Charm defined in vnf[id={}] but not present in "
568 "package".format(indata
["id"]))
569 vld_names
= [] # For detection of duplicated VLD names
570 for ivld
in get_iterable(indata
.get("internal-vld")):
571 # BEGIN Detection of duplicated VLD names
572 ivld_name
= ivld
["name"]
573 if ivld_name
in vld_names
:
574 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
575 .format(ivld
["name"], indata
["id"], ivld
["id"]),
576 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
578 vld_names
.append(ivld_name
)
579 # END Detection of duplicated VLD names
580 for icp
in get_iterable(ivld
.get("internal-connection-point")):
582 for vdu
in get_iterable(indata
.get("vdu")):
583 for internal_cp
in get_iterable(vdu
.get("internal-connection-point")):
584 if icp
["id-ref"] == internal_cp
["id"]:
590 raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
591 "vdu:internal-connection-point".format(ivld
["id"], icp
["id-ref"]),
592 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
593 if ivld
.get("ip-profile-ref"):
594 for ip_prof
in get_iterable(indata
.get("ip-profiles")):
595 if ip_prof
["name"] == get_iterable(ivld
.get("ip-profile-ref")):
598 raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
599 ivld
["id"], ivld
["ip-profile-ref"]),
600 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
601 for mp
in get_iterable(indata
.get("monitoring-param")):
602 if mp
.get("vdu-monitoring-param"):
604 for vdu
in get_iterable(indata
.get("vdu")):
605 for vmp
in get_iterable(vdu
.get("monitoring-param")):
606 if vmp
["id"] == mp
["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu
["id"] ==\
607 mp
["vdu-monitoring-param"]["vdu-ref"]:
613 raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
614 "defined at vdu[id='{}'] or vdu does not exist"
615 .format(mp
["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
616 mp
["vdu-monitoring-param"]["vdu-ref"]),
617 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
618 elif mp
.get("vdu-metric"):
620 for vdu
in get_iterable(indata
.get("vdu")):
621 if vdu
.get("vdu-configuration"):
622 for metric
in get_iterable(vdu
["vdu-configuration"].get("metrics")):
623 if metric
["name"] == mp
["vdu-metric"]["vdu-metric-name-ref"] and vdu
["id"] == \
624 mp
["vdu-metric"]["vdu-ref"]:
630 raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
631 "vdu[id='{}'] or vdu does not exist"
632 .format(mp
["vdu-metric"]["vdu-metric-name-ref"],
633 mp
["vdu-metric"]["vdu-ref"]),
634 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
636 for sgd
in get_iterable(indata
.get("scaling-group-descriptor")):
637 for sp
in get_iterable(sgd
.get("scaling-policy")):
638 for sc
in get_iterable(sp
.get("scaling-criteria")):
639 for mp
in get_iterable(indata
.get("monitoring-param")):
640 if mp
["id"] == get_iterable(sc
.get("vnf-monitoring-param-ref")):
643 raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
644 "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
645 .format(sgd
["name"], sc
["name"], sc
["vnf-monitoring-param-ref"]),
646 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
647 for sgd_vdu
in get_iterable(sgd
.get("vdu")):
649 for vdu
in get_iterable(indata
.get("vdu")):
650 if vdu
["id"] == sgd_vdu
["vdu-id-ref"]:
656 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
657 .format(sgd
["name"], sgd_vdu
["vdu-id-ref"]),
658 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
659 for sca
in get_iterable(sgd
.get("scaling-config-action")):
660 if not indata
.get("vnf-configuration"):
661 raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
662 "scaling-group-descriptor[name='{}']:scaling-config-action"
663 .format(sgd
["name"]),
664 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
665 for primitive
in get_iterable(indata
["vnf-configuration"].get("config-primitive")):
666 if primitive
["name"] == sca
["vnf-config-primitive-name-ref"]:
669 raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
670 "primitive-name-ref='{}' does not match any "
671 "vnf-configuration:config-primitive:name"
672 .format(sgd
["name"], sca
["vnf-config-primitive-name-ref"]),
673 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
676 def _validate_input_edit(self
, indata
, force
=False):
677 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
680 def _validate_package_folders(self
, storage_params
, folder
, file=None):
681 if not storage_params
or not storage_params
.get("pkg-dir"):
684 if self
.fs
.file_exists("{}_".format(storage_params
["folder"]), 'dir'):
685 f
= "{}_/{}/{}".format(storage_params
["folder"], storage_params
["pkg-dir"], folder
)
687 f
= "{}/{}/{}".format(storage_params
["folder"], storage_params
["pkg-dir"], folder
)
689 return self
.fs
.file_exists("{}/{}".format(f
, file), 'file')
691 if self
.fs
.file_exists(f
, 'dir'):
692 if self
.fs
.dir_ls(f
):
696 def delete_extra(self
, session
, _id
, db_content
, not_send_msg
=None):
698 Deletes associate file system storage (via super)
699 Deletes associated vnfpkgops from database.
700 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
701 :param _id: server internal id
702 :param db_content: The database content of the descriptor
704 :raises: FsException in case of error while deleting associated storage
706 super().delete_extra(session
, _id
, db_content
, not_send_msg
)
707 self
.db
.del_list("vnfpkgops", {"vnfPkgId": _id
})
710 class NsdTopic(DescriptorTopic
):
714 def __init__(self
, db
, fs
, msg
, auth
):
715 DescriptorTopic
.__init
__(self
, db
, fs
, msg
, auth
)
718 def _remove_envelop(indata
=None):
721 clean_indata
= indata
723 if clean_indata
.get('nsd:nsd-catalog'):
724 clean_indata
= clean_indata
['nsd:nsd-catalog']
725 elif clean_indata
.get('nsd-catalog'):
726 clean_indata
= clean_indata
['nsd-catalog']
727 if clean_indata
.get('nsd'):
728 if not isinstance(clean_indata
['nsd'], list) or len(clean_indata
['nsd']) != 1:
729 raise EngineException("'nsd' must be a list of only one element")
730 clean_indata
= clean_indata
['nsd'][0]
731 elif clean_indata
.get('nsd:nsd'):
732 if not isinstance(clean_indata
['nsd:nsd'], list) or len(clean_indata
['nsd:nsd']) != 1:
733 raise EngineException("'nsd:nsd' must be a list of only one element")
734 clean_indata
= clean_indata
['nsd:nsd'][0]
737 def _validate_input_new(self
, indata
, storage_params
, force
=False):
738 indata
= self
.pyangbind_validation("nsds", indata
, force
)
739 # Cross references validation in the descriptor
740 # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
741 for vld
in get_iterable(indata
.get("vld")):
742 if vld
.get("mgmt-network") and vld
.get("ip-profile-ref"):
743 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
744 " You cannot set an ip-profile when mgmt-network is True"
745 .format(vld
["id"]), http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
746 for vnfd_cp
in get_iterable(vld
.get("vnfd-connection-point-ref")):
747 for constituent_vnfd
in get_iterable(indata
.get("constituent-vnfd")):
748 if vnfd_cp
["member-vnf-index-ref"] == constituent_vnfd
["member-vnf-index"]:
749 if vnfd_cp
.get("vnfd-id-ref") and vnfd_cp
["vnfd-id-ref"] != constituent_vnfd
["vnfd-id-ref"]:
750 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
751 "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
752 " '{}'".format(vld
["id"], vnfd_cp
["vnfd-id-ref"],
753 constituent_vnfd
["member-vnf-index"],
754 constituent_vnfd
["vnfd-id-ref"]),
755 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
758 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
759 "does not match any constituent-vnfd:member-vnf-index"
760 .format(vld
["id"], vnfd_cp
["member-vnf-index-ref"]),
761 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
763 for fgd
in get_iterable(indata
.get("vnffgd")):
764 for cls
in get_iterable(fgd
.get("classifier")):
765 rspref
= cls
.get("rsp-id-ref")
766 for rsp
in get_iterable(fgd
.get("rsp")):
767 rspid
= rsp
.get("id")
768 if rspid
and rspref
and rspid
== rspref
:
771 raise EngineException(
772 "Error at vnffgd[id='{}']:classifier[id='{}']:rsp-id-ref '{}' does not match any rsp:id"
773 .format(fgd
["id"], cls
["id"], rspref
),
774 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
777 def _validate_input_edit(self
, indata
, force
=False):
778 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
781 def _check_descriptor_dependencies(self
, session
, descriptor
):
783 Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
784 connection points are ok
785 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
786 :param descriptor: descriptor to be inserted or edit
787 :return: None or raises exception
791 member_vnfd_index
= {}
792 if descriptor
.get("constituent-vnfd") and not session
["force"]:
793 for vnf
in descriptor
["constituent-vnfd"]:
794 vnfd_id
= vnf
["vnfd-id-ref"]
795 filter_q
= self
._get
_project
_filter
(session
)
796 filter_q
["id"] = vnfd_id
797 vnf_list
= self
.db
.get_list("vnfds", filter_q
)
799 raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
800 "existing vnfd".format(vnfd_id
), http_code
=HTTPStatus
.CONFLICT
)
801 # elif len(vnf_list) > 1:
802 # raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
803 # http_code=HTTPStatus.CONFLICT)
804 member_vnfd_index
[vnf
["member-vnf-index"]] = vnf_list
[0]
806 # Cross references validation in the descriptor and vnfd connection point validation
807 for vld
in get_iterable(descriptor
.get("vld")):
808 for referenced_vnfd_cp
in get_iterable(vld
.get("vnfd-connection-point-ref")):
809 # look if this vnfd contains this connection point
810 vnfd
= member_vnfd_index
.get(referenced_vnfd_cp
["member-vnf-index-ref"])
811 for vnfd_cp
in get_iterable(vnfd
.get("connection-point")):
812 if referenced_vnfd_cp
.get("vnfd-connection-point-ref") == vnfd_cp
["name"]:
815 raise EngineException(
816 "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
817 "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
818 .format(vld
["id"], referenced_vnfd_cp
["member-vnf-index-ref"],
819 referenced_vnfd_cp
["vnfd-connection-point-ref"], vnfd
["id"]),
820 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
)
822 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
823 super().check_conflict_on_edit(session
, final_content
, edit_content
, _id
)
825 self
._check
_descriptor
_dependencies
(session
, final_content
)
827 def check_conflict_on_del(self
, session
, _id
, db_content
):
829 Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
830 that NSD can be public and be used by other projects.
831 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
832 :param _id: nsd internal id
833 :param db_content: The database content of the _id
834 :return: None or raises EngineException with the conflict
838 descriptor
= db_content
839 descriptor_id
= descriptor
.get("id")
840 if not descriptor_id
: # empty nsd not uploaded
843 # check NSD used by NS
844 _filter
= self
._get
_project
_filter
(session
)
845 _filter
["nsd-id"] = _id
846 if self
.db
.get_list("nsrs", _filter
):
847 raise EngineException("There is at least one NS using this descriptor", http_code
=HTTPStatus
.CONFLICT
)
849 # check NSD referenced by NST
850 del _filter
["nsd-id"]
851 _filter
["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
852 if self
.db
.get_list("nsts", _filter
):
853 raise EngineException("There is at least one NetSlice Template referencing this descriptor",
854 http_code
=HTTPStatus
.CONFLICT
)
857 class NstTopic(DescriptorTopic
):
861 def __init__(self
, db
, fs
, msg
, auth
):
862 DescriptorTopic
.__init
__(self
, db
, fs
, msg
, auth
)
865 def _remove_envelop(indata
=None):
868 clean_indata
= indata
870 if clean_indata
.get('nst'):
871 if not isinstance(clean_indata
['nst'], list) or len(clean_indata
['nst']) != 1:
872 raise EngineException("'nst' must be a list only one element")
873 clean_indata
= clean_indata
['nst'][0]
874 elif clean_indata
.get('nst:nst'):
875 if not isinstance(clean_indata
['nst:nst'], list) or len(clean_indata
['nst:nst']) != 1:
876 raise EngineException("'nst:nst' must be a list only one element")
877 clean_indata
= clean_indata
['nst:nst'][0]
880 def _validate_input_edit(self
, indata
, force
=False):
881 # TODO validate with pyangbind, serialize
884 def _validate_input_new(self
, indata
, storage_params
, force
=False):
885 indata
= self
.pyangbind_validation("nsts", indata
, force
)
888 def _check_descriptor_dependencies(self
, session
, descriptor
):
890 Check that the dependent descriptors exist on a new descriptor or edition
891 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
892 :param descriptor: descriptor to be inserted or edit
893 :return: None or raises exception
895 if not descriptor
.get("netslice-subnet"):
897 for nsd
in descriptor
["netslice-subnet"]:
898 nsd_id
= nsd
["nsd-ref"]
899 filter_q
= self
._get
_project
_filter
(session
)
900 filter_q
["id"] = nsd_id
901 if not self
.db
.get_list("nsds", filter_q
):
902 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
903 "existing nsd".format(nsd_id
), http_code
=HTTPStatus
.CONFLICT
)
905 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
906 super().check_conflict_on_edit(session
, final_content
, edit_content
, _id
)
908 self
._check
_descriptor
_dependencies
(session
, final_content
)
910 def check_conflict_on_del(self
, session
, _id
, db_content
):
912 Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
913 that NST can be public and be used by other projects.
914 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
915 :param _id: nst internal id
916 :param db_content: The database content of the _id.
917 :return: None or raises EngineException with the conflict
919 # TODO: Check this method
922 # Get Network Slice Template from Database
923 _filter
= self
._get
_project
_filter
(session
)
924 _filter
["_admin.nst-id"] = _id
925 if self
.db
.get_list("nsis", _filter
):
926 raise EngineException("there is at least one Netslice Instance using this descriptor",
927 http_code
=HTTPStatus
.CONFLICT
)
930 class PduTopic(BaseTopic
):
933 schema_new
= pdu_new_schema
934 schema_edit
= pdu_edit_schema
936 def __init__(self
, db
, fs
, msg
, auth
):
937 BaseTopic
.__init
__(self
, db
, fs
, msg
, auth
)
940 def format_on_new(content
, project_id
=None, make_public
=False):
941 BaseTopic
.format_on_new(content
, project_id
=project_id
, make_public
=make_public
)
942 content
["_admin"]["onboardingState"] = "CREATED"
943 content
["_admin"]["operationalState"] = "ENABLED"
944 content
["_admin"]["usageState"] = "NOT_IN_USE"
946 def check_conflict_on_del(self
, session
, _id
, db_content
):
948 Check that there is not any vnfr that uses this PDU
949 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
950 :param _id: pdu internal id
951 :param db_content: The database content of the _id.
952 :return: None or raises EngineException with the conflict
957 _filter
= self
._get
_project
_filter
(session
)
958 _filter
["vdur.pdu-id"] = _id
959 if self
.db
.get_list("vnfrs", _filter
):
960 raise EngineException("There is at least one VNF using this PDU", http_code
=HTTPStatus
.CONFLICT
)
963 class VnfPkgOpTopic(BaseTopic
):
966 schema_new
= vnfpkgop_new_schema
969 def __init__(self
, db
, fs
, msg
, auth
):
970 BaseTopic
.__init
__(self
, db
, fs
, msg
, auth
)
972 def edit(self
, session
, _id
, indata
=None, kwargs
=None, content
=None):
973 raise EngineException("Method 'edit' not allowed for topic '{}'".format(self
.topic
),
974 HTTPStatus
.METHOD_NOT_ALLOWED
)
976 def delete(self
, session
, _id
, dry_run
=False):
977 raise EngineException("Method 'delete' not allowed for topic '{}'".format(self
.topic
),
978 HTTPStatus
.METHOD_NOT_ALLOWED
)
980 def delete_list(self
, session
, filter_q
=None):
981 raise EngineException("Method 'delete_list' not allowed for topic '{}'".format(self
.topic
),
982 HTTPStatus
.METHOD_NOT_ALLOWED
)
984 def new(self
, rollback
, session
, indata
=None, kwargs
=None, headers
=None):
986 Creates a new entry into database.
987 :param rollback: list to append created items at database in case a rollback may to be done
988 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
989 :param indata: data to be inserted
990 :param kwargs: used to override the indata descriptor
991 :param headers: http request headers
993 _id: identity of the inserted data.
996 self
._update
_input
_with
_kwargs
(indata
, kwargs
)
997 validate_input(indata
, self
.schema_new
)
998 vnfpkg_id
= indata
["vnfPkgId"]
999 filter_q
= BaseTopic
._get
_project
_filter
(session
)
1000 filter_q
["_id"] = vnfpkg_id
1001 vnfd
= self
.db
.get_one("vnfds", filter_q
)
1002 operation
= indata
["lcmOperationType"]
1003 kdu_name
= indata
["kdu_name"]
1004 for kdu
in vnfd
.get("kdu", []):
1005 if kdu
["name"] == kdu_name
:
1006 helm_chart
= kdu
.get("helm-chart")
1007 juju_bundle
= kdu
.get("juju-bundle")
1010 raise EngineException("Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id
, kdu_name
))
1012 indata
["helm-chart"] = helm_chart
1013 match
= fullmatch(r
"([^/]*)/([^/]*)", helm_chart
)
1014 repo_name
= match
.group(1) if match
else None
1016 indata
["juju-bundle"] = juju_bundle
1017 match
= fullmatch(r
"([^/]*)/([^/]*)", juju_bundle
)
1018 repo_name
= match
.group(1) if match
else None
1020 raise EngineException("Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']"
1021 .format(vnfpkg_id
, kdu_name
))
1024 filter_q
["name"] = repo_name
1025 repo
= self
.db
.get_one("k8srepos", filter_q
)
1026 k8srepo_id
= repo
.get("_id")
1027 k8srepo_url
= repo
.get("url")
1031 indata
["k8srepoId"] = k8srepo_id
1032 indata
["k8srepo_url"] = k8srepo_url
1033 vnfpkgop_id
= str(uuid4())
1036 "operationState": "PROCESSING",
1037 "vnfPkgId": vnfpkg_id
,
1038 "lcmOperationType": operation
,
1039 "isAutomaticInvocation": False,
1040 "isCancelPending": False,
1041 "operationParams": indata
,
1043 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id
,
1044 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id
,
1047 self
.format_on_new(vnfpkgop_desc
, session
["project_id"], make_public
=session
["public"])
1048 ctime
= vnfpkgop_desc
["_admin"]["created"]
1049 vnfpkgop_desc
["statusEnteredTime"] = ctime
1050 vnfpkgop_desc
["startTime"] = ctime
1051 self
.db
.create(self
.topic
, vnfpkgop_desc
)
1052 rollback
.append({"topic": self
.topic
, "_id": vnfpkgop_id
})
1053 self
.msg
.write(self
.topic_msg
, operation
, vnfpkgop_desc
)
1054 return vnfpkgop_id
, None