From: delacruzramo Date: Fri, 13 Sep 2019 10:24:22 +0000 (+0200) Subject: NBI Quotas X-Git-Tag: v7.0.0rc1~36 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FNBI.git;a=commitdiff_plain;h=32bab47c7fde8ae22795306723f3441ec544fa2b;ds=sidebyside NBI Quotas Change-Id: I6d9762b3d8eb3610c00355971ac9a0964bc1b212 Signed-off-by: delacruzramo --- diff --git a/osm_nbi/admin_topics.py b/osm_nbi/admin_topics.py index 5008c60..c198733 100644 --- a/osm_nbi/admin_topics.py +++ b/osm_nbi/admin_topics.py @@ -36,8 +36,8 @@ class UserTopic(BaseTopic): schema_edit = user_edit_schema multiproject = False - def __init__(self, db, fs, msg): - BaseTopic.__init__(self, db, fs, msg) + def __init__(self, db, fs, msg, auth): + BaseTopic.__init__(self, db, fs, msg, auth) @staticmethod def _get_project_filter(session): @@ -130,8 +130,8 @@ class ProjectTopic(BaseTopic): schema_edit = project_edit_schema multiproject = False - def __init__(self, db, fs, msg): - BaseTopic.__init__(self, db, fs, msg) + def __init__(self, db, fs, msg, auth): + BaseTopic.__init__(self, db, fs, msg, auth) @staticmethod def _get_project_filter(session): @@ -394,8 +394,8 @@ class UserTopicAuth(UserTopic): schema_edit = user_edit_schema def __init__(self, db, fs, msg, auth): - UserTopic.__init__(self, db, fs, msg) - self.auth = auth + UserTopic.__init__(self, db, fs, msg, auth) + # self.auth = auth def check_conflict_on_new(self, session, indata): """ @@ -707,8 +707,8 @@ class ProjectTopicAuth(ProjectTopic): schema_edit = project_edit_schema def __init__(self, db, fs, msg, auth): - ProjectTopic.__init__(self, db, fs, msg) - self.auth = auth + ProjectTopic.__init__(self, db, fs, msg, auth) + # self.auth = auth def check_conflict_on_new(self, session, indata): """ @@ -749,7 +749,7 @@ class ProjectTopicAuth(ProjectTopic): raise EngineException("You cannot rename project 'admin'", http_code=HTTPStatus.CONFLICT) # Check that project name is not used, regardless keystone already checks this - if self.auth.get_project_list(filter_q={"name": project_name}): + if project_name and self.auth.get_project_list(filter_q={"name": project_name}): raise EngineException("project '{}' is already used".format(project_name), HTTPStatus.CONFLICT) def check_conflict_on_del(self, session, _id, db_content): @@ -888,8 +888,7 @@ class ProjectTopicAuth(ProjectTopic): self.check_conflict_on_edit(session, content, indata, _id=_id) self.format_on_edit(content, indata) - if "name" in indata: - content["name"] = indata["name"] + deep_update_rfc7396(content, indata) self.auth.update_project(content["_id"], content) except ValidationError as e: raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY) @@ -903,8 +902,8 @@ class RoleTopicAuth(BaseTopic): multiproject = False def __init__(self, db, fs, msg, auth, ops): - BaseTopic.__init__(self, db, fs, msg) - self.auth = auth + BaseTopic.__init__(self, db, fs, msg, auth) + # self.auth = auth self.operations = ops # self.topic = "roles_operations" if isinstance(auth, AuthconnKeystone) else "roles" diff --git a/osm_nbi/authconn_keystone.py b/osm_nbi/authconn_keystone.py index a6cf0f0..f32cfe9 100644 --- a/osm_nbi/authconn_keystone.py +++ b/osm_nbi/authconn_keystone.py @@ -420,7 +420,8 @@ class AuthconnKeystone(Authconn): projects = [{ "name": project.name, "_id": project.id, - "_admin": project.to_dict().get("_admin", {}) # TODO: REVISE + "_admin": project.to_dict().get("_admin", {}), # TODO: REVISE + "quotas": project.to_dict().get("quotas", {}), # TODO: REVISE } for project in projects] if filter_q and filter_q.get("_id"): @@ -443,7 +444,9 @@ class AuthconnKeystone(Authconn): """ try: result = self.keystone.projects.create(project_info["name"], self.project_domain_name, - _admin=project_info["_admin"]) + _admin=project_info["_admin"], + quotas=project_info.get("quotas", {}) + ) return result.id except ClientException as e: # self.logger.exception("Error during project creation using keystone: {}".format(e)) @@ -478,7 +481,10 @@ class AuthconnKeystone(Authconn): :return: None """ try: - self.keystone.projects.update(project_id, name=project_info["name"], _admin=project_info["_admin"]) + self.keystone.projects.update(project_id, name=project_info["name"], + _admin=project_info["_admin"], + quotas=project_info.get("quotas", {}) + ) except ClientException as e: # self.logger.exception("Error during project update using keystone: {}".format(e)) raise AuthconnOperationException("Error during project update using Keystone: {}".format(e)) diff --git a/osm_nbi/base_topic.py b/osm_nbi/base_topic.py index 9688f60..c22833b 100644 --- a/osm_nbi/base_topic.py +++ b/osm_nbi/base_topic.py @@ -57,6 +57,8 @@ class BaseTopic: schema_edit = None # to_override multiproject = True # True if this Topic can be shared by several projects. Then it contains _admin.projects_read + default_quota = 500 + # Alternative ID Fields for some Topics alt_id_field = { "projects": "name", @@ -64,11 +66,12 @@ class BaseTopic: "roles": "name" } - def __init__(self, db, fs, msg): + def __init__(self, db, fs, msg, auth): self.db = db self.fs = fs self.msg = msg self.logger = logging.getLogger("nbi.engine") + self.auth = auth @staticmethod def id_field(topic, value): @@ -84,6 +87,29 @@ class BaseTopic: return {} return indata + def check_quota(self, session): + """ + Check whether topic quota is exceeded by the given project + Used by relevant topics' 'new' function to decide whether or not creation of the new item should be allowed + :param projects: projects (tuple) for which quota should be checked + :param override: boolean. If true, don't raise ValidationError even though quota be exceeded + :return: None + :raise: + DbException if project not found + ValidationError if quota exceeded and not overridden + """ + if session["force"] or session["admin"]: + return + projects = session["project_id"] + for project in projects: + proj = self.auth.get_project(project) + pid = proj["_id"] + quota = proj.get("quotas", {}).get(self.topic, self.default_quota) + count = self.db.count(self.topic, {"_admin.projects_read": pid}) + if count >= quota: + name = proj["name"] + raise ValidationError("{} quota ({}) exceeded for project {} ({})".format(self.topic, quota, name, pid)) + def _validate_input_new(self, input, force=False): """ Validates input user content for a new entry. It uses jsonschema. Some overrides will use pyangbind @@ -349,6 +375,9 @@ class BaseTopic: op_id: operation id if this is asynchronous, None otherwise """ try: + if self.multiproject: + self.check_quota(session) + content = self._remove_envelop(indata) # Override descriptor with query string kwargs diff --git a/osm_nbi/descriptor_topics.py b/osm_nbi/descriptor_topics.py index 0c9a3a4..d01dc13 100644 --- a/osm_nbi/descriptor_topics.py +++ b/osm_nbi/descriptor_topics.py @@ -33,8 +33,8 @@ __author__ = "Alfonso Tierno " class DescriptorTopic(BaseTopic): - def __init__(self, db, fs, msg): - BaseTopic.__init__(self, db, fs, msg) + def __init__(self, db, fs, msg, auth): + BaseTopic.__init__(self, db, fs, msg, auth) def check_conflict_on_edit(self, session, final_content, edit_content, _id): super().check_conflict_on_edit(session, final_content, edit_content, _id) @@ -121,6 +121,9 @@ class DescriptorTopic(BaseTopic): """ try: + # Check Quota + self.check_quota(session) + # _remove_envelop if indata: if "userDefinedData" in indata: @@ -394,8 +397,8 @@ class VnfdTopic(DescriptorTopic): topic = "vnfds" topic_msg = "vnfd" - def __init__(self, db, fs, msg): - DescriptorTopic.__init__(self, db, fs, msg) + def __init__(self, db, fs, msg, auth): + DescriptorTopic.__init__(self, db, fs, msg, auth) @staticmethod def _remove_envelop(indata=None): @@ -652,8 +655,8 @@ class NsdTopic(DescriptorTopic): topic = "nsds" topic_msg = "nsd" - def __init__(self, db, fs, msg): - DescriptorTopic.__init__(self, db, fs, msg) + def __init__(self, db, fs, msg, auth): + DescriptorTopic.__init__(self, db, fs, msg, auth) @staticmethod def _remove_envelop(indata=None): @@ -791,8 +794,8 @@ class NstTopic(DescriptorTopic): topic = "nsts" topic_msg = "nst" - def __init__(self, db, fs, msg): - DescriptorTopic.__init__(self, db, fs, msg) + def __init__(self, db, fs, msg, auth): + DescriptorTopic.__init__(self, db, fs, msg, auth) @staticmethod def _remove_envelop(indata=None): @@ -866,8 +869,8 @@ class PduTopic(BaseTopic): schema_new = pdu_new_schema schema_edit = pdu_edit_schema - def __init__(self, db, fs, msg): - BaseTopic.__init__(self, db, fs, msg) + def __init__(self, db, fs, msg, auth): + BaseTopic.__init__(self, db, fs, msg, auth) @staticmethod def format_on_new(content, project_id=None, make_public=False): diff --git a/osm_nbi/engine.py b/osm_nbi/engine.py index 8233c90..67a9233 100644 --- a/osm_nbi/engine.py +++ b/osm_nbi/engine.py @@ -148,13 +148,13 @@ class Engine(object): self.write_lock = Lock() # create one class per topic for topic, topic_class in self.map_from_topic_to_class.items(): - if self.auth and topic_class in (UserTopicAuth, ProjectTopicAuth): - self.map_topic[topic] = topic_class(self.db, self.fs, self.msg, self.auth) - elif self.auth and topic_class == RoleTopicAuth: + # if self.auth and topic_class in (UserTopicAuth, ProjectTopicAuth): + # self.map_topic[topic] = topic_class(self.db, self.fs, self.msg, self.auth) + if self.auth and topic_class == RoleTopicAuth: self.map_topic[topic] = topic_class(self.db, self.fs, self.msg, self.auth, self.operations) else: - self.map_topic[topic] = topic_class(self.db, self.fs, self.msg) + self.map_topic[topic] = topic_class(self.db, self.fs, self.msg, self.auth) self.map_topic["pm_jobs"] = PmJobsTopic(config["prometheus"].get("host"), config["prometheus"].get("port")) except (DbException, FsException, MsgException) as e: diff --git a/osm_nbi/instance_topics.py b/osm_nbi/instance_topics.py index d3a3b41..8c084e4 100644 --- a/osm_nbi/instance_topics.py +++ b/osm_nbi/instance_topics.py @@ -33,8 +33,8 @@ class NsrTopic(BaseTopic): topic_msg = "ns" schema_new = ns_instantiate - def __init__(self, db, fs, msg): - BaseTopic.__init__(self, db, fs, msg) + def __init__(self, db, fs, msg, auth): + BaseTopic.__init__(self, db, fs, msg, auth) def _check_descriptor_dependencies(self, session, descriptor): """ @@ -182,6 +182,9 @@ class NsrTopic(BaseTopic): """ try: + step = "checking quotas" + self.check_quota(session) + step = "validating input parameters" ns_request = self._remove_envelop(indata) # Override descriptor with query string kwargs @@ -391,8 +394,8 @@ class VnfrTopic(BaseTopic): topic = "vnfrs" topic_msg = None - def __init__(self, db, fs, msg): - BaseTopic.__init__(self, db, fs, msg) + def __init__(self, db, fs, msg, auth): + BaseTopic.__init__(self, db, fs, msg, auth) def delete(self, session, _id, dry_run=False): raise EngineException("Method delete called directly", HTTPStatus.INTERNAL_SERVER_ERROR) @@ -415,8 +418,8 @@ class NsLcmOpTopic(BaseTopic): "terminate": None, } - def __init__(self, db, fs, msg): - BaseTopic.__init__(self, db, fs, msg) + def __init__(self, db, fs, msg, auth): + BaseTopic.__init__(self, db, fs, msg, auth) def _check_ns_operation(self, session, nsr, operation, indata): """ @@ -845,9 +848,9 @@ class NsiTopic(BaseTopic): topic = "nsis" topic_msg = "nsi" - def __init__(self, db, fs, msg): - BaseTopic.__init__(self, db, fs, msg) - self.nsrTopic = NsrTopic(db, fs, msg) + def __init__(self, db, fs, msg, auth): + BaseTopic.__init__(self, db, fs, msg, auth) + self.nsrTopic = NsrTopic(db, fs, msg, auth) @staticmethod def _format_ns_request(ns_request): @@ -1006,6 +1009,9 @@ class NsiTopic(BaseTopic): """ try: + step = "checking quotas" + self.check_quota(session) + step = "" slice_request = self._remove_envelop(indata) # Override descriptor with query string kwargs @@ -1169,9 +1175,9 @@ class NsiLcmOpTopic(BaseTopic): "terminate": None } - def __init__(self, db, fs, msg): - BaseTopic.__init__(self, db, fs, msg) - self.nsi_NsLcmOpTopic = NsLcmOpTopic(self.db, self.fs, self.msg) + def __init__(self, db, fs, msg, auth): + BaseTopic.__init__(self, db, fs, msg, auth) + self.nsi_NsLcmOpTopic = NsLcmOpTopic(self.db, self.fs, self.msg, self.auth) def _check_nsi_operation(self, session, nsir, operation, indata): """ diff --git a/osm_nbi/validation.py b/osm_nbi/validation.py index 904abbd..6e43be5 100644 --- a/osm_nbi/validation.py +++ b/osm_nbi/validation.py @@ -631,6 +631,7 @@ user_edit_schema = { } # PROJECTS +topics_with_quota = ["vnfds", "nsds", "nsts", "pdus", "nsrs", "nsis", "vim_accounts", "wim_accounts", "sdns"] project_new_schema = { "$schema": "http://json-schema.org/draft-04/schema#", "title": "New project schema for administrators", @@ -638,6 +639,11 @@ project_new_schema = { "properties": { "name": shortname_schema, "admin": bool_schema, + "quotas": { + "type": "object", + "properties": {topic: integer0_schema for topic in topics_with_quota}, + "additionalProperties": False + }, }, "required": ["name"], "additionalProperties": False @@ -649,6 +655,11 @@ project_edit_schema = { "properties": { "admin": bool_schema, "name": shortname_schema, # To allow Project Name modification + "quotas": { + "type": "object", + "properties": {topic: {"oneOf": [integer0_schema, null_schema]} for topic in topics_with_quota}, + "additionalProperties": False + }, }, "additionalProperties": False, "minProperties": 1