Coverage for osm_mon/dashboarder/service.py: 0%
190 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-05-06 19:04 +0000
1# -*- coding: utf-8 -*-
3# Copyright 2021 Whitestack, LLC
4# *************************************************************
6# This file is part of OSM Monitoring module
7# All Rights Reserved to Whitestack, LLC
9# Licensed under the Apache License, Version 2.0 (the "License"); you may
10# not use this file except in compliance with the License. You may obtain
11# a copy of the License at
13# http://www.apache.org/licenses/LICENSE-2.0
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
17# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
18# License for the specific language governing permissions and limitations
19# under the License.
20# For those usages not covered by the Apache License, Version 2.0 please
21# contact: glavado@whitestack.com or fbravo@whitestack.com
22##
23import logging
25from osm_mon.core.common_db import CommonDbClient
26from osm_mon.core.config import Config
27from osm_mon.core.keystone import KeystoneConnection
28from osm_mon.dashboarder.backends.grafana import GrafanaBackend
29from osm_mon import __path__ as mon_path
30from osm_mon.core.utils import find_in_list, create_filter_from_nsr
31import re
33log = logging.getLogger(__name__)
36class DashboarderService:
37 def __init__(self, config: Config):
38 self.conf = config
39 self.common_db = CommonDbClient(self.conf)
40 self.grafana = GrafanaBackend(self.conf)
42 if bool(self.conf.get("keystone", "enabled")):
43 self.keystone = KeystoneConnection(self.conf)
44 else:
45 self.keystone = None
47 def create_dashboards(self):
48 # TODO lavado: migrate these methods to mongo change streams
49 # Lists all dashboards and OSM resources for later comparisons
50 datasource_name_substr = self.conf.get("prometheus-operator", "ds_name_substr")
51 prom_operator_port = self.conf.get("prometheus-operator", "port")
52 dashboard_uids = self.grafana.get_all_dashboard_uids()
53 datasource_names = self.grafana.get_all_datasource_names(datasource_name_substr)
54 osm_resource_uids = []
55 osm_datasource_names = []
56 projects = []
58 # Check if keystone is the auth/projects backend and get projects from there
59 if self.keystone:
60 try:
61 projects.extend(
62 map(
63 lambda project: {"_id": project.id, "name": project.name},
64 self.keystone.getProjects(),
65 )
66 )
67 except Exception:
68 log.error("Cannot retrieve projects from keystone")
69 else:
70 projects.extend(self.common_db.get_projects())
72 # Reads existing project list and creates a dashboard for each
73 for project in projects:
74 project_id = project["_id"]
75 # Collect Project IDs for periodical dashboard clean-up
76 osm_resource_uids.append(project_id)
77 dashboard_path = "{}/dashboarder/templates/project_scoped.json".format(
78 mon_path[0]
79 )
80 cnf_dashboard_path = "{}/dashboarder/templates/cnf_scoped.json".format(
81 mon_path[0]
82 )
83 if project_id not in dashboard_uids:
84 project_name = project["name"]
85 if project_name != "admin":
86 # Create project folder in Grafana only if user is not admin.
87 # Admin user's dashboard will be created in default folder
88 self.grafana.create_grafana_folders(project_name)
89 self.grafana.create_dashboard(project_id, project_name, dashboard_path)
90 log.debug("Created dashboard for Project: %s", project_id)
91 else:
92 log.debug("Dashboard already exists")
94 # Read existing k8s cluster list and creates a dashboard for each
95 k8sclusters = self.common_db.get_k8sclusters()
96 for k8scluster in k8sclusters:
97 k8scluster_id = k8scluster["_id"]
98 k8scluster_name = k8scluster["name"]
99 osm_resource_uids.append(k8scluster_id)
100 osm_datasource_names.append(
101 "{}-{}".format(datasource_name_substr, k8scluster_name)
102 )
103 if k8scluster_id not in dashboard_uids:
104 projects_read = k8scluster["_admin"]["projects_read"]
105 if len(projects_read) and projects_read[0] == project_id:
106 # Collect K8S Cluster IDs for periodical dashboard clean-up
107 k8scluster_address = k8scluster["credentials"]["clusters"][0][
108 "cluster"
109 ]["server"]
110 # Extract K8S Cluster ip from url
111 k8scluster_ip = re.findall(
112 r"://([\w\-\.]+)", k8scluster_address
113 )[0]
115 # prometheus-operator url
116 datasource_url = "http://{}:{}".format(
117 k8scluster_ip, prom_operator_port
118 )
120 # Create datsource for prometheus-operator in grafana
121 datasource_type = "prometheus"
122 datasource_name = "{}-{}".format(
123 datasource_name_substr, k8scluster_name
124 )
125 if datasource_name not in datasource_names:
126 self.grafana.create_datasource(
127 datasource_name, datasource_type, datasource_url
128 )
129 log.debug(
130 "Created datasource for k8scluster: %s", k8scluster_id
131 )
133 if project["name"] != "admin":
134 self.grafana.create_dashboard(
135 k8scluster_id,
136 k8scluster_name,
137 cnf_dashboard_path,
138 project_name=project["name"],
139 datasource_name=datasource_name,
140 )
141 else:
142 self.grafana.create_dashboard(
143 k8scluster_id,
144 k8scluster_name,
145 cnf_dashboard_path,
146 datasource_name=datasource_name,
147 )
148 log.debug("Created dashboard for k8scluster: %s", k8scluster_id)
149 else:
150 log.debug(
151 "Dashboard already exist for k8scluster: %s", k8scluster_id
152 )
154 # Reads existing NS list and creates a dashboard for each
155 # TODO lavado: only create for ACTIVE NSRs
156 nsrs = self.common_db.get_nsrs()
157 for nsr in nsrs:
158 nsr_id = nsr["_id"]
159 dashboard_path = "{}/dashboarder/templates/ns_scoped.json".format(
160 mon_path[0]
161 )
162 # Collect NS IDs for periodical dashboard clean-up
163 osm_resource_uids.append(nsr_id)
164 # Check if the NSR's VNFDs contain metrics
165 # Only one DF at the moment, support for this feature is comming in the future
166 vnfds_profiles = nsr["nsd"]["df"][0]["vnf-profile"]
167 for vnf_profile in vnfds_profiles:
168 try:
169 vnfd = self.common_db.get_vnfd_by_id(
170 vnf_profile["vnfd-id"], create_filter_from_nsr(nsr)
171 )
172 # If there are metrics, create dashboard (if exists)
173 if vnfd.get("vdu"):
174 vdu_found = find_in_list(
175 vnfd.get("vdu"),
176 lambda a_vdu: "monitoring-parameter" in a_vdu,
177 )
178 else:
179 vdu_found = None
180 if vdu_found:
181 if nsr_id not in dashboard_uids:
182 nsr_name = nsr["name"]
183 project_id = nsr["_admin"]["projects_read"][0]
184 try:
185 # Get project details from commondb
186 project_details = self.common_db.get_project(project_id)
187 project_name = project_details["name"]
188 except Exception as e:
189 # Project not found in commondb
190 if self.keystone:
191 # Serach project in keystone
192 for project in projects:
193 if project_id == project["_id"]:
194 project_name = project["name"]
195 else:
196 log.info("Project %s not found", project_id)
197 log.debug("Exception %s" % e)
198 self.grafana.create_dashboard(
199 nsr_id,
200 nsr_name,
201 dashboard_path,
202 project_name=project_name,
203 )
204 log.debug("Created dashboard for NS: %s", nsr_id)
205 else:
206 log.debug("Dashboard already exists")
207 break
208 else:
209 log.debug("NS does not has metrics")
210 except Exception:
211 log.exception("VNFD is not valid or has been renamed")
212 continue
214 # Delete obsolete dashboards
215 for dashboard_uid in dashboard_uids:
216 if dashboard_uid not in osm_resource_uids:
217 self.grafana.delete_dashboard(dashboard_uid)
218 log.debug("Deleted obsolete dashboard: %s", dashboard_uid)
219 else:
220 log.debug("All dashboards in use")
222 # Delete obsolute datasources
223 for datasource_name in datasource_names:
224 if datasource_name not in osm_datasource_names:
225 self.grafana.delete_datasource(datasource_name)
226 log.debug("Deleted obsolete datasource: %s", datasource_name)
227 else:
228 log.debug("All dashboards in use")
230 def create_grafana_user(self, user):
231 self.grafana.create_grafana_users(user)
233 def delete_non_existing_users(self):
234 if self.keystone:
235 # Get users from keystone
236 users = self.keystone.getUsers()
237 usernames = []
238 for user in users:
239 usernames.append(user.name)
240 grafana_users = self.grafana.get_grafana_users()
241 users_to_be_deleted = list(set(grafana_users) - set(usernames))
242 for grafana_user in users_to_be_deleted:
243 self.grafana.delete_grafana_users(grafana_user)
245 def create_grafana_team_member(
246 self, project_data, userid=None, project_list=None, user=None
247 ):
248 if user:
249 user_name = user
250 else:
251 try:
252 # Get user details from commondb
253 user = self.common_db.get_user_by_id(userid)
254 user_name = user["username"]
255 except Exception as e:
256 # User not found in commondb
257 if self.keystone:
258 # Search user in keystone
259 user = self.keystone.getUserById(userid)
260 user_name = user.name
261 else:
262 log.info("User %s not found", userid)
263 log.debug("Exception %s" % e)
264 if project_list:
265 # user-project mapping is done by osm cli
266 for proj in project_data:
267 project = self.common_db.get_project(proj["project"])
268 proj_name = project["name"]
269 role_obj = self.common_db.get_role_by_id(proj["role"])
270 is_admin = role_obj["permissions"].get("admin")
271 self.grafana.create_grafana_teams_members(
272 proj_name, user_name, is_admin, project_list
273 )
274 else:
275 # user-project mapping is done by osm ui
276 proj_list = []
277 if self.keystone:
278 users_proj_list = self.keystone.getProjectsById(userid)
279 for project in users_proj_list:
280 proj_list.append(project.name)
281 else:
282 users_proj_list = user.get("project_role_mappings")
283 for project in users_proj_list:
284 proj_data = self.common_db.get_project(project["project"])
285 proj_list.append(proj_data["name"])
286 for proj in project_data:
287 if self.keystone:
288 # Backend authentication type is keystone
289 try:
290 # Getting project and role objects from keystone using ids
291 role_obj = self.keystone.getRoleById(proj["role"])
292 proj_data = self.keystone.getProjectsByProjectId(
293 proj["project"]
294 )
295 log.info(
296 "role object {} {}".format(
297 role_obj.permissions, proj_data.name
298 )
299 )
300 is_admin = role_obj.permissions["admin"]
301 except Exception:
302 # Getting project and role objects from keystone using names
303 role_obj = self.keystone.getRoleByName(proj["role"])[0]
304 proj_data = self.keystone.getProjectsByProjectName(
305 proj["project"]
306 )[0]
307 is_admin = role_obj.to_dict().get("permissions").get("admin")
308 log.info(
309 "role object {} {}".format(
310 role_obj.to_dict(), proj_data.name
311 )
312 )
313 proj_name = proj_data.name
314 else:
315 # Backend authentication type is internal
316 try:
317 # Getting project and role object from commondb using names
318 role_obj = self.common_db.get_role_by_name(proj["role"])
319 proj_name = proj["project"]
320 except Exception:
321 # Getting project and role object from commondb using ids
322 role_obj = self.common_db.get_role_by_id(proj["role"])
323 proj_data = self.common_db.get_project(proj["project"])
324 proj_name = proj_data["name"]
325 is_admin = role_obj["permissions"].get("admin")
326 self.grafana.create_grafana_teams_members(
327 proj_name, user_name, is_admin, proj_list
328 )
330 def create_grafana_team(self, team_name):
331 self.grafana.create_grafana_teams(team_name)
333 def delete_grafana_user(self, user_name):
334 self.grafana.delete_grafana_users(user_name)
336 def delete_grafana_team(self, project_name):
337 self.grafana.delete_grafana_team(project_name)
339 def update_grafana_team(self, project_new_name, project_old_name):
340 self.grafana.update_grafana_teams(project_new_name, project_old_name)
342 def remove_grafana_team_members(self, user_id, proj_data):
343 try:
344 # Get user details from commondb
345 user = self.common_db.get_user_by_id(user_id)
346 user_name = user["username"]
347 except Exception as e:
348 # User not found in commondb
349 if self.keystone:
350 # Find user in keystone
351 user = self.keystone.getUserById(user_id)
352 user_name = user.name
353 else:
354 log.info("User %s not found", user_id)
355 log.debug("Exception %s" % e)
356 self.grafana.remove_grafana_team_member(user_name, proj_data)