From: palsus Date: Tue, 9 Feb 2021 18:23:03 +0000 (+0000) Subject: Fix for bug 1414, bug 1435 & bug 1438 - Multi-tenancy with keystone as backend X-Git-Tag: branch-sol006v331-start~18 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FMON.git;a=commitdiff_plain;h=0ab6407f2b68db44fbcf07ff2764153e2a896eaf Fix for bug 1414, bug 1435 & bug 1438 - Multi-tenancy with keystone as backend Change-Id: Ifd154f5ddb75897209c2281c96087840432fb110 Signed-off-by: palsus --- diff --git a/osm_mon/core/keystone.py b/osm_mon/core/keystone.py index d79ea89..98a8d78 100644 --- a/osm_mon/core/keystone.py +++ b/osm_mon/core/keystone.py @@ -56,3 +56,9 @@ class KeystoneConnection: Grabs projects from keystone using the client and session build in the constructor """ return self.keystone_client.projects.list() + + def getUserById(self, user_id): + """ + Grabs user object from keystone using user id + """ + return self.keystone_client.users.get(user_id) diff --git a/osm_mon/core/mon.yaml b/osm_mon/core/mon.yaml index 2ce2fc1..426675a 100644 --- a/osm_mon/core/mon.yaml +++ b/osm_mon/core/mon.yaml @@ -70,7 +70,7 @@ keystone: enabled: false url: http://keystone:5000/v3 domain_name: default - service_project: service - service_user: nbi - service_password: apassword + service_project: admin + service_user: admin + service_password: admin service_project_domain_name: default diff --git a/osm_mon/dashboarder/backends/grafana.py b/osm_mon/dashboarder/backends/grafana.py index c38a844..140a192 100644 --- a/osm_mon/dashboarder/backends/grafana.py +++ b/osm_mon/dashboarder/backends/grafana.py @@ -1,67 +1,67 @@ -# -*- coding: utf-8 -*- - -# Copyright 2018 Whitestack, LLC -# ************************************************************* - -# This file is part of OSM Monitoring module -# All Rights Reserved to Whitestack, LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# For those usages not covered by the Apache License, Version 2.0 please -# contact: glavado@whitestack.com or fbravo@whitestack.com -## -import logging -import requests -import base64 -import json -from osm_mon.core.config import Config - -log = logging.getLogger(__name__) - - -class GrafanaBackend: - def __init__(self, config: Config): - self.conf = config - self.url = config.get('grafana', 'url') - grafana_user = config.get("grafana", "user") - grafana_password = config.get("grafana", "password") - self.headers = { - 'content-type': "application/json", - 'authorization': "Basic %s" % base64.b64encode( - (grafana_user + ":" + grafana_password).encode("utf-8")).decode() - } - - def get_all_dashboard_uids(self): - # Gets only dashboards that were automated by OSM (with tag 'osm_automated') - response = requests.request("GET", self.url + "/api/search?tag=osm_automated", headers=self.headers) - dashboards = response.json() - dashboard_uids = [] - for dashboard in dashboards: - dashboard_uids.append(dashboard['uid']) - log.debug("Searching for all dashboard uids: %s", dashboard_uids) - return dashboard_uids - - def get_dashboard_status(self, uid): - response = requests.request("GET", self.url + "/api/dashboards/uid/" + uid, headers=self.headers) - log.debug("Searching for dashboard result: %s", response.text) - return response - - def create_dashboard(self, uid, name, json_file, project_name=None): - try: - with open(json_file) as f: - dashboard_data = f.read() - - dashboard_data = dashboard_data.replace('OSM_ID', uid).replace('OSM_NAME', name) +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# For those usages not covered by the Apache License, Version 2.0 please +# contact: glavado@whitestack.com or fbravo@whitestack.com +## +import logging +import requests +import base64 +import json +from osm_mon.core.config import Config + +log = logging.getLogger(__name__) + + +class GrafanaBackend: + def __init__(self, config: Config): + self.conf = config + self.url = config.get('grafana', 'url') + grafana_user = config.get("grafana", "user") + grafana_password = config.get("grafana", "password") + self.headers = { + 'content-type': "application/json", + 'authorization': "Basic %s" % base64.b64encode( + (grafana_user + ":" + grafana_password).encode("utf-8")).decode() + } + + def get_all_dashboard_uids(self): + # Gets only dashboards that were automated by OSM (with tag 'osm_automated') + response = requests.request("GET", self.url + "/api/search?tag=osm_automated", headers=self.headers) + dashboards = response.json() + dashboard_uids = [] + for dashboard in dashboards: + dashboard_uids.append(dashboard['uid']) + log.debug("Searching for all dashboard uids: %s", dashboard_uids) + return dashboard_uids + + def get_dashboard_status(self, uid): + response = requests.request("GET", self.url + "/api/dashboards/uid/" + uid, headers=self.headers) + log.debug("Searching for dashboard result: %s", response.text) + return response + + def create_dashboard(self, uid, name, json_file, project_name=None): + try: + with open(json_file) as f: + dashboard_data = f.read() + + dashboard_data = dashboard_data.replace('OSM_ID', uid).replace('OSM_NAME', name) dashboard_json_data = json.loads(dashboard_data) # get folder id if project_name: @@ -74,102 +74,108 @@ class GrafanaBackend: folder_id = json.loads(response_folder_id.text)["id"] dashboard_json_data["folderId"] = folder_id dashboard_json_data["overwrite"] = False - - response = requests.request( + + response = requests.request( "POST", self.url + "/api/dashboards/db/", data=json.dumps(dashboard_json_data), headers=self.headers) - # get team id - if project_name is not None: - name = project_name - response_team = requests.request( - "GET", self.url + "/api/teams/search?name={}".format(name), headers=self.headers) - if len(json.loads(response_team.text)["teams"]) > 0: - team_id = json.loads(response_team.text)["teams"][0]["id"] - permission_data = {"items": [{"teamId": team_id, "permission": 2}, ]} - # provide permission to dashboard - response = requests.request( + # get team id + if project_name is not None: + name = project_name + response_team = requests.request( + "GET", self.url + "/api/teams/search?name={}".format(name), headers=self.headers) + if len(json.loads(response_team.text)["teams"]) > 0: + team_id = json.loads(response_team.text)["teams"][0]["id"] + permission_data = {"items": [{"teamId": team_id, "permission": 2}, ]} + # provide permission to dashboard + response = requests.request( "POST", self.url + "/api/folders/{}/permissions".format(project_name), json=permission_data, - headers=self.headers) + headers=self.headers) log.info("Dashboard %s is created in Grafana", name) - return response - except Exception: - log.exception("Exception processing message: ") - - def delete_dashboard(self, uid): - response = requests.request("DELETE", self.url + "/api/dashboards/uid/" + uid, headers=self.headers) + return response + except Exception: + log.exception("Exception processing message: ") + + def delete_dashboard(self, uid): + response = requests.request("DELETE", self.url + "/api/dashboards/uid/" + uid, headers=self.headers) log.debug("Dashboard %s deleted from Grafana", uid) - return response - - def create_grafana_users(self, user): - email = "{}@osm.etsi.org".format(user) - user_payload = { - "name": user, - "email": email, - "login": user, - "password": user, - } - response_users = requests.request("POST", self.url + "/api/admin/users/", json=user_payload, - headers=self.headers) - json_data = json.loads(response_users.text) - url = "/api/org/users/{}/".format(json_data["id"]) - permission_payload = {"role": "Editor", } + return response + + def create_grafana_users(self, user): + email = "{}@osm.etsi.org".format(user) + user_payload = { + "name": user, + "email": email, + "login": user, + "password": user, + } + response_users = requests.request("POST", self.url + "/api/admin/users/", json=user_payload, + headers=self.headers) + json_data = json.loads(response_users.text) + url = "/api/org/users/{}/".format(json_data["id"]) + permission_payload = {"role": "Editor", } requests.request("PATCH", self.url + url, json=permission_payload, headers=self.headers) log.info("New user %s created in Grafana", user) - return response_users - - # create grafana team with member - def create_grafana_teams_members(self, project_name, user_name, is_admin, proj_list): - # check if user exist in grafana or not - user_response = requests.request("GET", self.url + "/api/users/lookup?loginOrEmail={}".format(user_name), - headers=self.headers) - user_obj = json.loads(user_response.text) - if user_response.status_code != 200: - user_response = self.create_grafana_users(user_name) - user_obj = json.loads(user_response.text) - - user_id = user_obj["id"] - - # Get Teams for user - team_objs = requests.request("GET", self.url + "/api/users/{}/teams".format(user_id), headers=self.headers) - team_obj = json.loads(team_objs.text) - team_list = [] - if len(team_obj): - for team in team_obj: - team_list.append(team["name"]) - - proj_unlink = set(team_list) - set(proj_list) - for prj in proj_unlink: - response_team = requests.request("GET", self.url + "/api/teams/search?name={}".format(prj), - headers=self.headers) - team_id = json.loads(response_team.text)["teams"][0]["id"] - requests.request("DELETE", self.url + "/api/teams/{}/members/{}".format(team_id, user_id), - headers=self.headers) - # add member to team - response_team = requests.request("GET", self.url + "/api/teams/search?name={}".format(project_name), - headers=self.headers) - team_id = json.loads(response_team.text)["teams"][0]["id"] - if project_name not in team_list: - member_payload = { - "userId": user_id - } - requests.request("POST", self.url + "/api/teams/{}/members".format(team_id), json=member_payload, - headers=self.headers) - # if role is admin change permission to admin - if is_admin: - url = "/api/org/users/{}/".format(user_id) - permission_payload = {"role": "Admin", } - requests.request("PATCH", self.url + url, json=permission_payload, headers=self.headers) + return response_users + + # create grafana team with member + def create_grafana_teams_members(self, project_name, user_name, is_admin, proj_list): + # check if user exist in grafana or not + user_response = requests.request("GET", self.url + "/api/users/lookup?loginOrEmail={}".format(user_name), + headers=self.headers) + user_obj = json.loads(user_response.text) + if user_response.status_code != 200: + user_response = self.create_grafana_users(user_name) + user_obj = json.loads(user_response.text) + + user_id = user_obj["id"] + + # Get Teams for user + team_objs = requests.request("GET", self.url + "/api/users/{}/teams".format(user_id), headers=self.headers) + team_obj = json.loads(team_objs.text) + team_list = [] + if len(team_obj): + for team in team_obj: + team_list.append(team["name"]) + + proj_unlink = set(team_list) - set(proj_list) + for prj in proj_unlink: + response_team = requests.request("GET", self.url + "/api/teams/search?name={}".format(prj), + headers=self.headers) + team_id = json.loads(response_team.text)["teams"][0]["id"] + requests.request("DELETE", self.url + "/api/teams/{}/members/{}".format(team_id, user_id), + headers=self.headers) + if project_name != "admin": + # add member to team + response_team = requests.request("GET", self.url + "/api/teams/search?name={}".format(project_name), + headers=self.headers) + + if not json.loads(response_team.text)["teams"]: + # team doesn't exist in grafana. Creating it first + self.create_grafana_teams(project_name) + response_team = requests.request("GET", self.url + "/api/teams/search?name={}".format(project_name), + headers=self.headers) + team_id = json.loads(response_team.text)["teams"][0]["id"] + if project_name not in team_list: + member_payload = { + "userId": user_id + } + requests.request("POST", self.url + "/api/teams/{}/members".format(team_id), json=member_payload, + headers=self.headers) + # if role is admin change permission to admin + if is_admin: + url = "/api/org/users/{}/".format(user_id) + permission_payload = {"role": "Admin", } + requests.request("PATCH", self.url + url, json=permission_payload, headers=self.headers) log.info("User %s is assigned Admin permission", user_name) else: url = "/api/org/users/{}/".format(user_id) permission_payload = {"role": "Editor", } requests.request("PATCH", self.url + url, json=permission_payload, headers=self.headers) log.info("User %s is assigned Editor permission", user_name) - return response_team - - # create grafana team - def create_grafana_teams(self, team_name): - team_payload = {"name": team_name, } + + # create grafana team + def create_grafana_teams(self, team_name): + team_payload = {"name": team_name, } requests.request("POST", self.url + "/api/teams", json=team_payload, headers=self.headers) log.info("New team %s created in Grafana", team_name) @@ -178,37 +184,37 @@ class GrafanaBackend: folder_payload = {"uid": folder_name, "title": folder_name} requests.request("POST", self.url + "/api/folders", json=folder_payload, headers=self.headers) log.info("Dashboard folder %s created", folder_name) - - def delete_grafana_users(self, user_name): - # find user id - response_id = requests.request("GET", self.url + "/api/users/lookup?loginOrEmail={}".format(user_name), - headers=self.headers) - try: - user_id = json.loads(response_id.text)["id"] - except Exception: - log.exception("Exception processing message: ") - # delete user - response = requests.request("DELETE", self.url + "/api/admin/users/{}".format(user_id), headers=self.headers) + + def delete_grafana_users(self, user_name): + # find user id + response_id = requests.request("GET", self.url + "/api/users/lookup?loginOrEmail={}".format(user_name), + headers=self.headers) + try: + user_id = json.loads(response_id.text)["id"] + except Exception: + log.exception("Exception processing message: ") + # delete user + response = requests.request("DELETE", self.url + "/api/admin/users/{}".format(user_id), headers=self.headers) log.info("User %s deleted in Grafana", user_name) - return response - - def delete_grafana_team(self, project_name): + return response + + def delete_grafana_team(self, project_name): # delete grafana folder requests.request("DELETE", self.url + "/api/folders/{}".format(project_name), headers=self.headers) # delete grafana team - team_obj = requests.request("GET", self.url + "/api/teams/search?name={}".format(project_name), - headers=self.headers) - team_id = json.loads(team_obj.text)["teams"][0]["id"] - response = requests.request("DELETE", self.url + "/api/teams/{}".format(team_id), headers=self.headers) + team_obj = requests.request("GET", self.url + "/api/teams/search?name={}".format(project_name), + headers=self.headers) + team_id = json.loads(team_obj.text)["teams"][0]["id"] + response = requests.request("DELETE", self.url + "/api/teams/{}".format(team_id), headers=self.headers) log.info("Team %s deleted in Grafana", project_name) - return response - - def update_grafana_teams(self, project_new_name, project_old_name): - team_obj = requests.request("GET", self.url + "/api/teams/search?name={}".format(project_old_name), - headers=self.headers) - team_id = json.loads(team_obj.text)["teams"][0]["id"] - data = {"name": project_new_name, } - response = requests.request("PUT", self.url + "/api/teams/{}".format(team_id), json=data, headers=self.headers) - log.info("Grafana team updated %s", response.text) + return response + + def update_grafana_teams(self, project_new_name, project_old_name): + team_obj = requests.request("GET", self.url + "/api/teams/search?name={}".format(project_old_name), + headers=self.headers) + team_id = json.loads(team_obj.text)["teams"][0]["id"] + data = {"name": project_new_name, } + response = requests.request("PUT", self.url + "/api/teams/{}".format(team_id), json=data, headers=self.headers) + log.info("Grafana team updated %s", response.text) return response diff --git a/osm_mon/dashboarder/service.py b/osm_mon/dashboarder/service.py index 32128c2..0dd3d90 100644 --- a/osm_mon/dashboarder/service.py +++ b/osm_mon/dashboarder/service.py @@ -68,7 +68,10 @@ class DashboarderService: dashboard_path = '{}/dashboarder/templates/project_scoped.json'.format(mon_path[0]) if project_id not in dashboard_uids: project_name = project['name'] - self.grafana.create_grafana_folders(project_name) + if project_name != "admin": + # Create project folder in Grafana only if user is not admin. + # Admin user's dashboard will be created in default folder + self.grafana.create_grafana_folders(project_name) self.grafana.create_dashboard(project_id, project_name, dashboard_path) log.debug('Created dashboard for Project: %s', project_id) @@ -95,8 +98,20 @@ class DashboarderService: if nsr_id not in dashboard_uids: nsr_name = nsr['name'] project_id = nsr["_admin"]["projects_read"][0] - project_details = self.common_db.get_project(project_id) - project_name = project_details["name"] + try: + # Get project details from commondb + project_details = self.common_db.get_project(project_id) + project_name = project_details["name"] + except Exception as e: + # Project not found in commondb + if self.keystone: + # Serach project in keystone + for project in projects: + if project_id == project['_id']: + project_name = project["name"] + else: + log.info('Project %s not found', project_id) + log.debug('Exception %s' % e) self.grafana.create_dashboard(nsr_id, nsr_name, dashboard_path, project_name) log.debug('Created dashboard for NS: %s', nsr_id) @@ -121,8 +136,19 @@ class DashboarderService: self.grafana.create_grafana_users(user) def create_grafana_team_member(self, project_data, userid): - user = self.common_db.get_user_by_id(userid) - user_name = user["username"] + try: + # Get user details from commondb + user = self.common_db.get_user_by_id(userid) + user_name = user["username"] + except Exception as e: + # User not found in commondb + if self.keystone: + # Serach user in keystone + user = self.keystone.getUserById(userid) + user_name = user.name + else: + log.info('User %s not found', userid) + log.debug('Exception %s' % e) proj_list = [] for project in project_data: proj_list.append(project["project"])