Fix bug 1701: remove vcaId parameter in NS deployment
[osm/NBI.git] / osm_nbi / base_topic.py
1 # -*- coding: utf-8 -*-
2
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12 # implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15
16 import logging
17 from uuid import uuid4
18 from http import HTTPStatus
19 from time import time
20 from osm_common.dbbase import deep_update_rfc7396
21 from osm_nbi.validation import validate_input, ValidationError, is_valid_uuid
22 from yaml import safe_load, YAMLError
23
24 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
25
26
27 class EngineException(Exception):
28 def __init__(self, message, http_code=HTTPStatus.BAD_REQUEST):
29 self.http_code = http_code
30 super(Exception, self).__init__(message)
31
32
33 def deep_get(target_dict, key_list):
34 """
35 Get a value from target_dict entering in the nested keys. If keys does not exist, it returns None
36 Example target_dict={a: {b: 5}}; key_list=[a,b] returns 5; both key_list=[a,b,c] and key_list=[f,h] return None
37 :param target_dict: dictionary to be read
38 :param key_list: list of keys to read from target_dict
39 :return: The wanted value if exist, None otherwise
40 """
41 for key in key_list:
42 if not isinstance(target_dict, dict) or key not in target_dict:
43 return None
44 target_dict = target_dict[key]
45 return target_dict
46
47
48 def get_iterable(input_var):
49 """
50 Returns an iterable, in case input_var is None it just returns an empty tuple
51 :param input_var: can be a list, tuple or None
52 :return: input_var or () if it is None
53 """
54 if input_var is None:
55 return ()
56 return input_var
57
58
59 def versiontuple(v):
60 """utility for compare dot separate versions. Fills with zeros to proper number comparison"""
61 filled = []
62 for point in v.split("."):
63 filled.append(point.zfill(8))
64 return tuple(filled)
65
66
67 def increment_ip_mac(ip_mac, vm_index=1):
68 if not isinstance(ip_mac, str):
69 return ip_mac
70 try:
71 # try with ipv4 look for last dot
72 i = ip_mac.rfind(".")
73 if i > 0:
74 i += 1
75 return "{}{}".format(ip_mac[:i], int(ip_mac[i:]) + vm_index)
76 # try with ipv6 or mac look for last colon. Operate in hex
77 i = ip_mac.rfind(":")
78 if i > 0:
79 i += 1
80 # format in hex, len can be 2 for mac or 4 for ipv6
81 return ("{}{:0" + str(len(ip_mac) - i) + "x}").format(
82 ip_mac[:i], int(ip_mac[i:], 16) + vm_index
83 )
84 except Exception:
85 pass
86 return None
87
88
89 class BaseTopic:
90 # static variables for all instance classes
91 topic = None # to_override
92 topic_msg = None # to_override
93 quota_name = None # to_override. If not provided topic will be used for quota_name
94 schema_new = None # to_override
95 schema_edit = None # to_override
96 multiproject = True # True if this Topic can be shared by several projects. Then it contains _admin.projects_read
97
98 default_quota = 500
99
100 # Alternative ID Fields for some Topics
101 alt_id_field = {"projects": "name", "users": "username", "roles": "name"}
102
103 def __init__(self, db, fs, msg, auth):
104 self.db = db
105 self.fs = fs
106 self.msg = msg
107 self.logger = logging.getLogger("nbi.engine")
108 self.auth = auth
109
110 @staticmethod
111 def id_field(topic, value):
112 """Returns ID Field for given topic and field value"""
113 if topic in BaseTopic.alt_id_field.keys() and not is_valid_uuid(value):
114 return BaseTopic.alt_id_field[topic]
115 else:
116 return "_id"
117
118 @staticmethod
119 def _remove_envelop(indata=None):
120 if not indata:
121 return {}
122 return indata
123
124 def check_quota(self, session):
125 """
126 Check whether topic quota is exceeded by the given project
127 Used by relevant topics' 'new' function to decide whether or not creation of the new item should be allowed
128 :param session[project_id]: projects (tuple) for which quota should be checked
129 :param session[force]: boolean. If true, skip quota checking
130 :return: None
131 :raise:
132 DbException if project not found
133 ValidationError if quota exceeded in one of the projects
134 """
135 if session["force"]:
136 return
137 projects = session["project_id"]
138 for project in projects:
139 proj = self.auth.get_project(project)
140 pid = proj["_id"]
141 quota_name = self.quota_name or self.topic
142 quota = proj.get("quotas", {}).get(quota_name, self.default_quota)
143 count = self.db.count(self.topic, {"_admin.projects_read": pid})
144 if count >= quota:
145 name = proj["name"]
146 raise ValidationError(
147 "quota ({}={}) exceeded for project {} ({})".format(
148 quota_name, quota, name, pid
149 ),
150 http_code=HTTPStatus.UNPROCESSABLE_ENTITY,
151 )
152
153 def _validate_input_new(self, input, force=False):
154 """
155 Validates input user content for a new entry. It uses jsonschema. Some overrides will use pyangbind
156 :param input: user input content for the new topic
157 :param force: may be used for being more tolerant
158 :return: The same input content, or a changed version of it.
159 """
160 if self.schema_new:
161 validate_input(input, self.schema_new)
162 return input
163
164 def _validate_input_edit(self, input, content, force=False):
165 """
166 Validates input user content for an edition. It uses jsonschema. Some overrides will use pyangbind
167 :param input: user input content for the new topic
168 :param force: may be used for being more tolerant
169 :return: The same input content, or a changed version of it.
170 """
171 if self.schema_edit:
172 validate_input(input, self.schema_edit)
173 return input
174
175 @staticmethod
176 def _get_project_filter(session):
177 """
178 Generates a filter dictionary for querying database, so that only allowed items for this project can be
179 addressed. Only proprietary or public can be used. Allowed projects are at _admin.project_read/write. If it is
180 not present or contains ANY mean public.
181 :param session: contains:
182 project_id: project list this session has rights to access. Can be empty, one or several
183 set_project: items created will contain this project list
184 force: True or False
185 public: True, False or None
186 method: "list", "show", "write", "delete"
187 admin: True or False
188 :return: dictionary with project filter
189 """
190 p_filter = {}
191 project_filter_n = []
192 project_filter = list(session["project_id"])
193
194 if session["method"] not in ("list", "delete"):
195 if project_filter:
196 project_filter.append("ANY")
197 elif session["public"] is not None:
198 if session["public"]:
199 project_filter.append("ANY")
200 else:
201 project_filter_n.append("ANY")
202
203 if session.get("PROJECT.ne"):
204 project_filter_n.append(session["PROJECT.ne"])
205
206 if project_filter:
207 if session["method"] in ("list", "show", "delete") or session.get(
208 "set_project"
209 ):
210 p_filter["_admin.projects_read.cont"] = project_filter
211 else:
212 p_filter["_admin.projects_write.cont"] = project_filter
213 if project_filter_n:
214 if session["method"] in ("list", "show", "delete") or session.get(
215 "set_project"
216 ):
217 p_filter["_admin.projects_read.ncont"] = project_filter_n
218 else:
219 p_filter["_admin.projects_write.ncont"] = project_filter_n
220
221 return p_filter
222
223 def check_conflict_on_new(self, session, indata):
224 """
225 Check that the data to be inserted is valid
226 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
227 :param indata: data to be inserted
228 :return: None or raises EngineException
229 """
230 pass
231
232 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
233 """
234 Check that the data to be edited/uploaded is valid
235 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
236 :param final_content: data once modified. This method may change it.
237 :param edit_content: incremental data that contains the modifications to apply
238 :param _id: internal _id
239 :return: final_content or raises EngineException
240 """
241 if not self.multiproject:
242 return final_content
243 # Change public status
244 if session["public"] is not None:
245 if (
246 session["public"]
247 and "ANY" not in final_content["_admin"]["projects_read"]
248 ):
249 final_content["_admin"]["projects_read"].append("ANY")
250 final_content["_admin"]["projects_write"].clear()
251 if (
252 not session["public"]
253 and "ANY" in final_content["_admin"]["projects_read"]
254 ):
255 final_content["_admin"]["projects_read"].remove("ANY")
256
257 # Change project status
258 if session.get("set_project"):
259 for p in session["set_project"]:
260 if p not in final_content["_admin"]["projects_read"]:
261 final_content["_admin"]["projects_read"].append(p)
262
263 return final_content
264
265 def check_unique_name(self, session, name, _id=None):
266 """
267 Check that the name is unique for this project
268 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
269 :param name: name to be checked
270 :param _id: If not None, ignore this entry that are going to change
271 :return: None or raises EngineException
272 """
273 if not self.multiproject:
274 _filter = {}
275 else:
276 _filter = self._get_project_filter(session)
277 _filter["name"] = name
278 if _id:
279 _filter["_id.neq"] = _id
280 if self.db.get_one(
281 self.topic, _filter, fail_on_empty=False, fail_on_more=False
282 ):
283 raise EngineException(
284 "name '{}' already exists for {}".format(name, self.topic),
285 HTTPStatus.CONFLICT,
286 )
287
288 @staticmethod
289 def format_on_new(content, project_id=None, make_public=False):
290 """
291 Modifies content descriptor to include _admin
292 :param content: descriptor to be modified
293 :param project_id: if included, it add project read/write permissions. Can be None or a list
294 :param make_public: if included it is generated as public for reading.
295 :return: op_id: operation id on asynchronous operation, None otherwise. In addition content is modified
296 """
297 now = time()
298 if "_admin" not in content:
299 content["_admin"] = {}
300 if not content["_admin"].get("created"):
301 content["_admin"]["created"] = now
302 content["_admin"]["modified"] = now
303 if not content.get("_id"):
304 content["_id"] = str(uuid4())
305 if project_id is not None:
306 if not content["_admin"].get("projects_read"):
307 content["_admin"]["projects_read"] = list(project_id)
308 if make_public:
309 content["_admin"]["projects_read"].append("ANY")
310 if not content["_admin"].get("projects_write"):
311 content["_admin"]["projects_write"] = list(project_id)
312 return None
313
314 @staticmethod
315 def format_on_edit(final_content, edit_content):
316 """
317 Modifies final_content to admin information upon edition
318 :param final_content: final content to be stored at database
319 :param edit_content: user requested update content
320 :return: operation id, if this edit implies an asynchronous operation; None otherwise
321 """
322 if final_content.get("_admin"):
323 now = time()
324 final_content["_admin"]["modified"] = now
325 return None
326
327 def _send_msg(self, action, content, not_send_msg=None):
328 if self.topic_msg and not_send_msg is not False:
329 content = content.copy()
330 content.pop("_admin", None)
331 if isinstance(not_send_msg, list):
332 not_send_msg.append((self.topic_msg, action, content))
333 else:
334 self.msg.write(self.topic_msg, action, content)
335
336 def check_conflict_on_del(self, session, _id, db_content):
337 """
338 Check if deletion can be done because of dependencies if it is not force. To override
339 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
340 :param _id: internal _id
341 :param db_content: The database content of this item _id
342 :return: None if ok or raises EngineException with the conflict
343 """
344 pass
345
346 @staticmethod
347 def _update_input_with_kwargs(desc, kwargs, yaml_format=False):
348 """
349 Update descriptor with the kwargs. It contains dot separated keys
350 :param desc: dictionary to be updated
351 :param kwargs: plain dictionary to be used for updating.
352 :param yaml_format: get kwargs values as yaml format.
353 :return: None, 'desc' is modified. It raises EngineException.
354 """
355 if not kwargs:
356 return
357 try:
358 for k, v in kwargs.items():
359 update_content = desc
360 kitem_old = None
361 klist = k.split(".")
362 for kitem in klist:
363 if kitem_old is not None:
364 update_content = update_content[kitem_old]
365 if isinstance(update_content, dict):
366 kitem_old = kitem
367 if not isinstance(update_content.get(kitem_old), (dict, list)):
368 update_content[kitem_old] = {}
369 elif isinstance(update_content, list):
370 # key must be an index of the list, must be integer
371 kitem_old = int(kitem)
372 # if index greater than list, extend the list
373 if kitem_old >= len(update_content):
374 update_content += [None] * (
375 kitem_old - len(update_content) + 1
376 )
377 if not isinstance(update_content[kitem_old], (dict, list)):
378 update_content[kitem_old] = {}
379 else:
380 raise EngineException(
381 "Invalid query string '{}'. Descriptor is not a list nor dict at '{}'".format(
382 k, kitem
383 )
384 )
385 if v is None:
386 del update_content[kitem_old]
387 else:
388 update_content[kitem_old] = v if not yaml_format else safe_load(v)
389 except KeyError:
390 raise EngineException(
391 "Invalid query string '{}'. Descriptor does not contain '{}'".format(
392 k, kitem_old
393 )
394 )
395 except ValueError:
396 raise EngineException(
397 "Invalid query string '{}'. Expected integer index list instead of '{}'".format(
398 k, kitem
399 )
400 )
401 except IndexError:
402 raise EngineException(
403 "Invalid query string '{}'. Index '{}' out of range".format(
404 k, kitem_old
405 )
406 )
407 except YAMLError:
408 raise EngineException("Invalid query string '{}' yaml format".format(k))
409
410 def sol005_projection(self, data):
411 # Projection was moved to child classes
412 return data
413
414 def show(self, session, _id, filter_q=None, api_req=False):
415 """
416 Get complete information on an topic
417 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
418 :param _id: server internal id
419 :param filter_q: dict: query parameter
420 :param api_req: True if this call is serving an external API request. False if serving internal request.
421 :return: dictionary, raise exception if not found.
422 """
423 if not self.multiproject:
424 filter_db = {}
425 else:
426 filter_db = self._get_project_filter(session)
427 # To allow project&user addressing by name AS WELL AS _id
428 filter_db[BaseTopic.id_field(self.topic, _id)] = _id
429 data = self.db.get_one(self.topic, filter_db)
430
431 # Only perform SOL005 projection if we are serving an external request
432 if api_req:
433 self.sol005_projection(data)
434
435 return data
436
437 # TODO transform data for SOL005 URL requests
438 # TODO remove _admin if not admin
439
440 def get_file(self, session, _id, path=None, accept_header=None):
441 """
442 Only implemented for descriptor topics. Return the file content of a descriptor
443 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
444 :param _id: Identity of the item to get content
445 :param path: artifact path or "$DESCRIPTOR" or None
446 :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
447 :return: opened file or raises an exception
448 """
449 raise EngineException(
450 "Method get_file not valid for this topic", HTTPStatus.INTERNAL_SERVER_ERROR
451 )
452
453 def list(self, session, filter_q=None, api_req=False):
454 """
455 Get a list of the topic that matches a filter
456 :param session: contains the used login username and working project
457 :param filter_q: filter of data to be applied
458 :param api_req: True if this call is serving an external API request. False if serving internal request.
459 :return: The list, it can be empty if no one match the filter.
460 """
461 if not filter_q:
462 filter_q = {}
463 if self.multiproject:
464 filter_q.update(self._get_project_filter(session))
465
466 # TODO transform data for SOL005 URL requests. Transform filtering
467 # TODO implement "field-type" query string SOL005
468 data = self.db.get_list(self.topic, filter_q)
469
470 # Only perform SOL005 projection if we are serving an external request
471 if api_req:
472 data = [self.sol005_projection(inst) for inst in data]
473
474 return data
475
476 def new(self, rollback, session, indata=None, kwargs=None, headers=None):
477 """
478 Creates a new entry into database.
479 :param rollback: list to append created items at database in case a rollback may to be done
480 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
481 :param indata: data to be inserted
482 :param kwargs: used to override the indata descriptor
483 :param headers: http request headers
484 :return: _id, op_id:
485 _id: identity of the inserted data.
486 op_id: operation id if this is asynchronous, None otherwise
487 """
488 try:
489 if self.multiproject:
490 self.check_quota(session)
491
492 content = self._remove_envelop(indata)
493
494 # Override descriptor with query string kwargs
495 self._update_input_with_kwargs(content, kwargs)
496 content = self._validate_input_new(content, force=session["force"])
497 self.check_conflict_on_new(session, content)
498 op_id = self.format_on_new(
499 content, project_id=session["project_id"], make_public=session["public"]
500 )
501 _id = self.db.create(self.topic, content)
502 rollback.append({"topic": self.topic, "_id": _id})
503 if op_id:
504 content["op_id"] = op_id
505 self._send_msg("created", content)
506 return _id, op_id
507 except ValidationError as e:
508 raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
509
510 def upload_content(self, session, _id, indata, kwargs, headers):
511 """
512 Only implemented for descriptor topics. Used for receiving content by chunks (with a transaction_id header
513 and/or gzip file. It will store and extract)
514 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
515 :param _id : the database id of entry to be updated
516 :param indata: http body request
517 :param kwargs: user query string to override parameters. NOT USED
518 :param headers: http request headers
519 :return: True package has is completely uploaded or False if partial content has been uplodaed.
520 Raise exception on error
521 """
522 raise EngineException(
523 "Method upload_content not valid for this topic",
524 HTTPStatus.INTERNAL_SERVER_ERROR,
525 )
526
527 def delete_list(self, session, filter_q=None):
528 """
529 Delete a several entries of a topic. This is for internal usage and test only, not exposed to NBI API
530 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
531 :param filter_q: filter of data to be applied
532 :return: The deleted list, it can be empty if no one match the filter.
533 """
534 # TODO add admin to filter, validate rights
535 if not filter_q:
536 filter_q = {}
537 if self.multiproject:
538 filter_q.update(self._get_project_filter(session))
539 return self.db.del_list(self.topic, filter_q)
540
541 def delete_extra(self, session, _id, db_content, not_send_msg=None):
542 """
543 Delete other things apart from database entry of a item _id.
544 e.g.: other associated elements at database and other file system storage
545 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
546 :param _id: server internal id
547 :param db_content: The database content of the _id. It is already deleted when reached this method, but the
548 content is needed in same cases
549 :param not_send_msg: To not send message (False) or store content (list) instead
550 :return: None if ok or raises EngineException with the problem
551 """
552 pass
553
554 def delete(self, session, _id, dry_run=False, not_send_msg=None):
555 """
556 Delete item by its internal _id
557 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
558 :param _id: server internal id
559 :param dry_run: make checking but do not delete
560 :param not_send_msg: To not send message (False) or store content (list) instead
561 :return: operation id (None if there is not operation), raise exception if error or not found, conflict, ...
562 """
563
564 # To allow addressing projects and users by name AS WELL AS by _id
565 if not self.multiproject:
566 filter_q = {}
567 else:
568 filter_q = self._get_project_filter(session)
569 filter_q[self.id_field(self.topic, _id)] = _id
570 item_content = self.db.get_one(self.topic, filter_q)
571
572 self.check_conflict_on_del(session, _id, item_content)
573 if dry_run:
574 return None
575
576 if self.multiproject and session["project_id"]:
577 # remove reference from project_read if there are more projects referencing it. If it last one,
578 # do not remove reference, but delete
579 other_projects_referencing = next(
580 (
581 p
582 for p in item_content["_admin"]["projects_read"]
583 if p not in session["project_id"] and p != "ANY"
584 ),
585 None,
586 )
587
588 # check if there are projects referencing it (apart from ANY, that means, public)....
589 if other_projects_referencing:
590 # remove references but not delete
591 update_dict_pull = {
592 "_admin.projects_read": session["project_id"],
593 "_admin.projects_write": session["project_id"],
594 }
595 self.db.set_one(
596 self.topic, filter_q, update_dict=None, pull_list=update_dict_pull
597 )
598 return None
599 else:
600 can_write = next(
601 (
602 p
603 for p in item_content["_admin"]["projects_write"]
604 if p == "ANY" or p in session["project_id"]
605 ),
606 None,
607 )
608 if not can_write:
609 raise EngineException(
610 "You have not write permission to delete it",
611 http_code=HTTPStatus.UNAUTHORIZED,
612 )
613
614 # delete
615 self.db.del_one(self.topic, filter_q)
616 self.delete_extra(session, _id, item_content, not_send_msg=not_send_msg)
617 self._send_msg("deleted", {"_id": _id}, not_send_msg=not_send_msg)
618 return None
619
620 def edit(self, session, _id, indata=None, kwargs=None, content=None):
621 """
622 Change the content of an item
623 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
624 :param _id: server internal id
625 :param indata: contains the changes to apply
626 :param kwargs: modifies indata
627 :param content: original content of the item
628 :return: op_id: operation id if this is processed asynchronously, None otherwise
629 """
630 indata = self._remove_envelop(indata)
631
632 # Override descriptor with query string kwargs
633 if kwargs:
634 self._update_input_with_kwargs(indata, kwargs)
635 try:
636 if indata and session.get("set_project"):
637 raise EngineException(
638 "Cannot edit content and set to project (query string SET_PROJECT) at same time",
639 HTTPStatus.UNPROCESSABLE_ENTITY,
640 )
641 # TODO self._check_edition(session, indata, _id, force)
642 if not content:
643 content = self.show(session, _id)
644 indata = self._validate_input_edit(indata, content, force=session["force"])
645 deep_update_rfc7396(content, indata)
646
647 # To allow project addressing by name AS WELL AS _id. Get the _id, just in case the provided one is a name
648 _id = content.get("_id") or _id
649
650 content = self.check_conflict_on_edit(session, content, indata, _id=_id)
651 op_id = self.format_on_edit(content, indata)
652
653 self.db.replace(self.topic, _id, content)
654
655 indata.pop("_admin", None)
656 if op_id:
657 indata["op_id"] = op_id
658 indata["_id"] = _id
659 self._send_msg("edited", indata)
660 return op_id
661 except ValidationError as e:
662 raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)