1 # -*- coding: utf-8 -*-
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
17 from uuid
import uuid4
18 from http
import HTTPStatus
20 from osm_common
.dbbase
import deep_update_rfc7396
21 from osm_nbi
.validation
import validate_input
, ValidationError
, is_valid_uuid
23 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
26 class EngineException(Exception):
28 def __init__(self
, message
, http_code
=HTTPStatus
.BAD_REQUEST
):
29 self
.http_code
= http_code
30 super(Exception, self
).__init
__(message
)
33 def deep_get(target_dict
, key_list
):
35 Get a value from target_dict entering in the nested keys. If keys does not exist, it returns None
36 Example target_dict={a: {b: 5}}; key_list=[a,b] returns 5; both key_list=[a,b,c] and key_list=[f,h] return None
37 :param target_dict: dictionary to be read
38 :param key_list: list of keys to read from target_dict
39 :return: The wanted value if exist, None otherwise
42 if not isinstance(target_dict
, dict) or key
not in target_dict
:
44 target_dict
= target_dict
[key
]
48 def get_iterable(input_var
):
50 Returns an iterable, in case input_var is None it just returns an empty tuple
51 :param input_var: can be a list, tuple or None
52 :return: input_var or () if it is None
60 """utility for compare dot separate versions. Fills with zeros to proper number comparison"""
62 for point
in v
.split("."):
63 filled
.append(point
.zfill(8))
68 # static variables for all instance classes
69 topic
= None # to_override
70 topic_msg
= None # to_override
71 schema_new
= None # to_override
72 schema_edit
= None # to_override
73 multiproject
= True # True if this Topic can be shared by several projects. Then it contains _admin.projects_read
77 # Alternative ID Fields for some Topics
84 def __init__(self
, db
, fs
, msg
, auth
):
88 self
.logger
= logging
.getLogger("nbi.engine")
92 def id_field(topic
, value
):
93 """Returns ID Field for given topic and field value"""
94 if topic
in BaseTopic
.alt_id_field
.keys() and not is_valid_uuid(value
):
95 return BaseTopic
.alt_id_field
[topic
]
100 def _remove_envelop(indata
=None):
105 def check_quota(self
, session
):
107 Check whether topic quota is exceeded by the given project
108 Used by relevant topics' 'new' function to decide whether or not creation of the new item should be allowed
109 :param projects: projects (tuple) for which quota should be checked
110 :param override: boolean. If true, don't raise ValidationError even though quota be exceeded
113 DbException if project not found
114 ValidationError if quota exceeded and not overridden
116 if session
["force"] or session
["admin"]:
118 projects
= session
["project_id"]
119 for project
in projects
:
120 proj
= self
.auth
.get_project(project
)
122 quota
= proj
.get("quotas", {}).get(self
.topic
, self
.default_quota
)
123 count
= self
.db
.count(self
.topic
, {"_admin.projects_read": pid
})
126 raise ValidationError("{} quota ({}) exceeded for project {} ({})".format(self
.topic
, quota
, name
, pid
))
128 def _validate_input_new(self
, input, force
=False):
130 Validates input user content for a new entry. It uses jsonschema. Some overrides will use pyangbind
131 :param input: user input content for the new topic
132 :param force: may be used for being more tolerant
133 :return: The same input content, or a changed version of it.
136 validate_input(input, self
.schema_new
)
139 def _validate_input_edit(self
, input, force
=False):
141 Validates input user content for an edition. It uses jsonschema. Some overrides will use pyangbind
142 :param input: user input content for the new topic
143 :param force: may be used for being more tolerant
144 :return: The same input content, or a changed version of it.
147 validate_input(input, self
.schema_edit
)
151 def _get_project_filter(session
):
153 Generates a filter dictionary for querying database, so that only allowed items for this project can be
154 addressed. Only propietary or public can be used. Allowed projects are at _admin.project_read/write. If it is
155 not present or contains ANY mean public.
156 :param session: contains:
157 project_id: project list this session has rights to access. Can be empty, one or several
158 set_project: items created will contain this project list
160 public: True, False or None
161 method: "list", "show", "write", "delete"
163 :return: dictionary with project filter
166 project_filter_n
= []
167 project_filter
= list(session
["project_id"])
169 if session
["method"] not in ("list", "delete"):
171 project_filter
.append("ANY")
172 elif session
["public"] is not None:
173 if session
["public"]:
174 project_filter
.append("ANY")
176 project_filter_n
.append("ANY")
178 if session
.get("PROJECT.ne"):
179 project_filter_n
.append(session
["PROJECT.ne"])
182 if session
["method"] in ("list", "show", "delete") or session
.get("set_project"):
183 p_filter
["_admin.projects_read.cont"] = project_filter
185 p_filter
["_admin.projects_write.cont"] = project_filter
187 if session
["method"] in ("list", "show", "delete") or session
.get("set_project"):
188 p_filter
["_admin.projects_read.ncont"] = project_filter_n
190 p_filter
["_admin.projects_write.ncont"] = project_filter_n
194 def check_conflict_on_new(self
, session
, indata
):
196 Check that the data to be inserted is valid
197 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
198 :param indata: data to be inserted
199 :return: None or raises EngineException
203 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
205 Check that the data to be edited/uploaded is valid
206 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
207 :param final_content: data once modified. This method may change it.
208 :param edit_content: incremental data that contains the modifications to apply
209 :param _id: internal _id
210 :return: None or raises EngineException
212 if not self
.multiproject
:
214 # Change public status
215 if session
["public"] is not None:
216 if session
["public"] and "ANY" not in final_content
["_admin"]["projects_read"]:
217 final_content
["_admin"]["projects_read"].append("ANY")
218 final_content
["_admin"]["projects_write"].clear()
219 if not session
["public"] and "ANY" in final_content
["_admin"]["projects_read"]:
220 final_content
["_admin"]["projects_read"].remove("ANY")
222 # Change project status
223 if session
.get("set_project"):
224 for p
in session
["set_project"]:
225 if p
not in final_content
["_admin"]["projects_read"]:
226 final_content
["_admin"]["projects_read"].append(p
)
228 def check_unique_name(self
, session
, name
, _id
=None):
230 Check that the name is unique for this project
231 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
232 :param name: name to be checked
233 :param _id: If not None, ignore this entry that are going to change
234 :return: None or raises EngineException
236 if not self
.multiproject
:
239 _filter
= self
._get
_project
_filter
(session
)
240 _filter
["name"] = name
242 _filter
["_id.neq"] = _id
243 if self
.db
.get_one(self
.topic
, _filter
, fail_on_empty
=False, fail_on_more
=False):
244 raise EngineException("name '{}' already exists for {}".format(name
, self
.topic
), HTTPStatus
.CONFLICT
)
247 def format_on_new(content
, project_id
=None, make_public
=False):
249 Modifies content descriptor to include _admin
250 :param content: descriptor to be modified
251 :param project_id: if included, it add project read/write permissions. Can be None or a list
252 :param make_public: if included it is generated as public for reading.
253 :return: op_id: operation id on asynchronous operation, None otherwise. In addition content is modified
256 if "_admin" not in content
:
257 content
["_admin"] = {}
258 if not content
["_admin"].get("created"):
259 content
["_admin"]["created"] = now
260 content
["_admin"]["modified"] = now
261 if not content
.get("_id"):
262 content
["_id"] = str(uuid4())
263 if project_id
is not None:
264 if not content
["_admin"].get("projects_read"):
265 content
["_admin"]["projects_read"] = list(project_id
)
267 content
["_admin"]["projects_read"].append("ANY")
268 if not content
["_admin"].get("projects_write"):
269 content
["_admin"]["projects_write"] = list(project_id
)
273 def format_on_edit(final_content
, edit_content
):
275 Modifies final_content to admin information upon edition
276 :param final_content: final content to be stored at database
277 :param edit_content: user requested update content
278 :return: operation id, if this edit implies an asynchronous operation; None otherwise
280 if final_content
.get("_admin"):
282 final_content
["_admin"]["modified"] = now
285 def _send_msg(self
, action
, content
):
287 content
.pop("_admin", None)
288 self
.msg
.write(self
.topic_msg
, action
, content
)
290 def check_conflict_on_del(self
, session
, _id
, db_content
):
292 Check if deletion can be done because of dependencies if it is not force. To override
293 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
294 :param _id: internal _id
295 :param db_content: The database content of this item _id
296 :return: None if ok or raises EngineException with the conflict
301 def _update_input_with_kwargs(desc
, kwargs
):
303 Update descriptor with the kwargs. It contains dot separated keys
304 :param desc: dictionary to be updated
305 :param kwargs: plain dictionary to be used for updating.
306 :return: None, 'desc' is modified. It raises EngineException.
311 for k
, v
in kwargs
.items():
312 update_content
= desc
316 if kitem_old
is not None:
317 update_content
= update_content
[kitem_old
]
318 if isinstance(update_content
, dict):
320 elif isinstance(update_content
, list):
321 kitem_old
= int(kitem
)
323 raise EngineException(
324 "Invalid query string '{}'. Descriptor is not a list nor dict at '{}'".format(k
, kitem
))
325 update_content
[kitem_old
] = v
327 raise EngineException(
328 "Invalid query string '{}'. Descriptor does not contain '{}'".format(k
, kitem_old
))
330 raise EngineException("Invalid query string '{}'. Expected integer index list instead of '{}'".format(
333 raise EngineException(
334 "Invalid query string '{}'. Index '{}' out of range".format(k
, kitem_old
))
336 def show(self
, session
, _id
):
338 Get complete information on an topic
339 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
340 :param _id: server internal id
341 :return: dictionary, raise exception if not found.
343 if not self
.multiproject
:
346 filter_db
= self
._get
_project
_filter
(session
)
347 # To allow project&user addressing by name AS WELL AS _id
348 filter_db
[BaseTopic
.id_field(self
.topic
, _id
)] = _id
349 return self
.db
.get_one(self
.topic
, filter_db
)
350 # TODO transform data for SOL005 URL requests
351 # TODO remove _admin if not admin
353 def get_file(self
, session
, _id
, path
=None, accept_header
=None):
355 Only implemented for descriptor topics. Return the file content of a descriptor
356 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
357 :param _id: Identity of the item to get content
358 :param path: artifact path or "$DESCRIPTOR" or None
359 :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
360 :return: opened file or raises an exception
362 raise EngineException("Method get_file not valid for this topic", HTTPStatus
.INTERNAL_SERVER_ERROR
)
364 def list(self
, session
, filter_q
=None):
366 Get a list of the topic that matches a filter
367 :param session: contains the used login username and working project
368 :param filter_q: filter of data to be applied
369 :return: The list, it can be empty if no one match the filter.
373 if self
.multiproject
:
374 filter_q
.update(self
._get
_project
_filter
(session
))
376 # TODO transform data for SOL005 URL requests. Transform filtering
377 # TODO implement "field-type" query string SOL005
378 return self
.db
.get_list(self
.topic
, filter_q
)
380 def new(self
, rollback
, session
, indata
=None, kwargs
=None, headers
=None):
382 Creates a new entry into database.
383 :param rollback: list to append created items at database in case a rollback may to be done
384 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
385 :param indata: data to be inserted
386 :param kwargs: used to override the indata descriptor
387 :param headers: http request headers
389 _id: identity of the inserted data.
390 op_id: operation id if this is asynchronous, None otherwise
393 if self
.multiproject
:
394 self
.check_quota(session
)
396 content
= self
._remove
_envelop
(indata
)
398 # Override descriptor with query string kwargs
399 self
._update
_input
_with
_kwargs
(content
, kwargs
)
400 content
= self
._validate
_input
_new
(content
, force
=session
["force"])
401 self
.check_conflict_on_new(session
, content
)
402 op_id
= self
.format_on_new(content
, project_id
=session
["project_id"], make_public
=session
["public"])
403 _id
= self
.db
.create(self
.topic
, content
)
404 rollback
.append({"topic": self
.topic
, "_id": _id
})
406 content
["op_id"] = op_id
407 self
._send
_msg
("created", content
)
409 except ValidationError
as e
:
410 raise EngineException(e
, HTTPStatus
.UNPROCESSABLE_ENTITY
)
412 def upload_content(self
, session
, _id
, indata
, kwargs
, headers
):
414 Only implemented for descriptor topics. Used for receiving content by chunks (with a transaction_id header
415 and/or gzip file. It will store and extract)
416 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
417 :param _id : the database id of entry to be updated
418 :param indata: http body request
419 :param kwargs: user query string to override parameters. NOT USED
420 :param headers: http request headers
421 :return: True package has is completely uploaded or False if partial content has been uplodaed.
422 Raise exception on error
424 raise EngineException("Method upload_content not valid for this topic", HTTPStatus
.INTERNAL_SERVER_ERROR
)
426 def delete_list(self
, session
, filter_q
=None):
428 Delete a several entries of a topic. This is for internal usage and test only, not exposed to NBI API
429 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
430 :param filter_q: filter of data to be applied
431 :return: The deleted list, it can be empty if no one match the filter.
433 # TODO add admin to filter, validate rights
436 if self
.multiproject
:
437 filter_q
.update(self
._get
_project
_filter
(session
))
438 return self
.db
.del_list(self
.topic
, filter_q
)
440 def delete_extra(self
, session
, _id
, db_content
):
442 Delete other things apart from database entry of a item _id.
443 e.g.: other associated elements at database and other file system storage
444 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
445 :param _id: server internal id
446 :param db_content: The database content of the _id. It is already deleted when reached this method, but the
447 content is needed in same cases
448 :return: None if ok or raises EngineException with the problem
452 def delete(self
, session
, _id
, dry_run
=False):
454 Delete item by its internal _id
455 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
456 :param _id: server internal id
457 :param dry_run: make checking but do not delete
458 :return: operation id (None if there is not operation), raise exception if error or not found, conflict, ...
461 # To allow addressing projects and users by name AS WELL AS by _id
462 filter_q
= {BaseTopic
.id_field(self
.topic
, _id
): _id
}
463 item_content
= self
.db
.get_one(self
.topic
, filter_q
)
465 # TODO add admin to filter, validate rights
466 # data = self.get_item(topic, _id)
467 self
.check_conflict_on_del(session
, _id
, item_content
)
471 if self
.multiproject
:
472 filter_q
.update(self
._get
_project
_filter
(session
))
473 if self
.multiproject
and session
["project_id"]:
474 # remove reference from project_read. If not last delete
475 # if this topic is not part of session["project_id"] no midification at database is done and an exception
477 self
.db
.set_one(self
.topic
, filter_q
, update_dict
=None,
478 pull
={"_admin.projects_read": {"$in": session
["project_id"]}})
479 # try to delete if there is not any more reference from projects. Ignore if it is not deleted
480 filter_q
= {'_id': _id
, '_admin.projects_read': [[], ["ANY"]]}
481 v
= self
.db
.del_one(self
.topic
, filter_q
, fail_on_empty
=False)
482 if not v
or not v
["deleted"]:
485 self
.db
.del_one(self
.topic
, filter_q
)
486 self
.delete_extra(session
, _id
, item_content
)
487 self
._send
_msg
("deleted", {"_id": _id
})
490 def edit(self
, session
, _id
, indata
=None, kwargs
=None, content
=None):
492 Change the content of an item
493 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
494 :param _id: server internal id
495 :param indata: contains the changes to apply
496 :param kwargs: modifies indata
497 :param content: original content of the item
498 :return: op_id: operation id if this is processed asynchronously, None otherwise
500 indata
= self
._remove
_envelop
(indata
)
502 # Override descriptor with query string kwargs
504 self
._update
_input
_with
_kwargs
(indata
, kwargs
)
506 if indata
and session
.get("set_project"):
507 raise EngineException("Cannot edit content and set to project (query string SET_PROJECT) at same time",
508 HTTPStatus
.UNPROCESSABLE_ENTITY
)
509 indata
= self
._validate
_input
_edit
(indata
, force
=session
["force"])
511 # TODO self._check_edition(session, indata, _id, force)
513 content
= self
.show(session
, _id
)
514 deep_update_rfc7396(content
, indata
)
516 # To allow project addressing by name AS WELL AS _id. Get the _id, just in case the provided one is a name
517 _id
= content
.get("_id") or _id
519 self
.check_conflict_on_edit(session
, content
, indata
, _id
=_id
)
520 op_id
= self
.format_on_edit(content
, indata
)
522 self
.db
.replace(self
.topic
, _id
, content
)
524 indata
.pop("_admin", None)
526 indata
["op_id"] = op_id
528 self
._send
_msg
("edited", indata
)
530 except ValidationError
as e
:
531 raise EngineException(e
, HTTPStatus
.UNPROCESSABLE_ENTITY
)