5c05ef657518f505ecff4d60824d6387118c1287
1 # -*- coding: utf-8 -*-
3 # Copyright 2021 Whitestack, LLC
4 # *************************************************************
6 # This file is part of OSM Monitoring module
7 # All Rights Reserved to Whitestack, LLC
9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
10 # not use this file except in compliance with the License. You may obtain
11 # a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
18 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact: glavado@whitestack.com or fbravo@whitestack.com
25 from osm_mon
.core
.common_db
import CommonDbClient
26 from osm_mon
.core
.config
import Config
27 from osm_mon
.core
.keystone
import KeystoneConnection
28 from osm_mon
.dashboarder
.backends
.grafana
import GrafanaBackend
29 from osm_mon
import __path__
as mon_path
30 from osm_mon
.core
.utils
import find_in_list
, create_filter_from_nsr
33 log
= logging
.getLogger(__name__
)
36 class DashboarderService
:
37 def __init__(self
, config
: Config
):
39 self
.common_db
= CommonDbClient(self
.conf
)
40 self
.grafana
= GrafanaBackend(self
.conf
)
42 if bool(self
.conf
.get("keystone", "enabled")):
43 self
.keystone
= KeystoneConnection(self
.conf
)
47 def create_dashboards(self
):
48 # TODO lavado: migrate these methods to mongo change streams
49 # Lists all dashboards and OSM resources for later comparisons
50 datasource_name_substr
= self
.conf
.get("prometheus-operator", "ds_name_substr")
51 prom_operator_port
= self
.conf
.get("prometheus-operator", "port")
52 dashboard_uids
= self
.grafana
.get_all_dashboard_uids()
53 datasource_names
= self
.grafana
.get_all_datasource_names(datasource_name_substr
)
54 osm_resource_uids
= []
55 osm_datasource_names
= []
58 # Check if keystone is the auth/projects backend and get projects from there
63 lambda project
: {"_id": project
.id, "name": project
.name
},
64 self
.keystone
.getProjects(),
68 log
.error("Cannot retrieve projects from keystone")
70 projects
.extend(self
.common_db
.get_projects())
72 # Reads existing project list and creates a dashboard for each
73 for project
in projects
:
74 project_id
= project
["_id"]
75 # Collect Project IDs for periodical dashboard clean-up
76 osm_resource_uids
.append(project_id
)
77 dashboard_path
= "{}/dashboarder/templates/project_scoped.json".format(
80 cnf_dashboard_path
= "{}/dashboarder/templates/cnf_scoped.json".format(
83 if project_id
not in dashboard_uids
:
84 project_name
= project
["name"]
85 if project_name
!= "admin":
86 # Create project folder in Grafana only if user is not admin.
87 # Admin user's dashboard will be created in default folder
88 self
.grafana
.create_grafana_folders(project_name
)
89 self
.grafana
.create_dashboard(project_id
, project_name
, dashboard_path
)
90 log
.debug("Created dashboard for Project: %s", project_id
)
92 log
.debug("Dashboard already exists")
94 # Read existing k8s cluster list and creates a dashboard for each
95 k8sclusters
= self
.common_db
.get_k8sclusters()
96 for k8scluster
in k8sclusters
:
97 k8scluster_id
= k8scluster
["_id"]
98 k8scluster_name
= k8scluster
["name"]
99 osm_resource_uids
.append(k8scluster_id
)
100 osm_datasource_names
.append(
101 "{}-{}".format(datasource_name_substr
, k8scluster_name
)
103 if k8scluster_id
not in dashboard_uids
:
104 projects_read
= k8scluster
["_admin"]["projects_read"]
105 if len(projects_read
) and projects_read
[0] == project_id
:
106 # Collect K8S Cluster IDs for periodical dashboard clean-up
107 k8scluster_address
= k8scluster
["credentials"]["clusters"][0][
110 # Extract K8S Cluster ip from url
111 k8scluster_ip
= re
.findall(
112 r
"://([\w\-\.]+)", k8scluster_address
115 # prometheus-operator url
116 datasource_url
= "http://{}:{}".format(
117 k8scluster_ip
, prom_operator_port
120 # Create datsource for prometheus-operator in grafana
121 datasource_type
= "prometheus"
122 datasource_name
= "{}-{}".format(
123 datasource_name_substr
, k8scluster_name
125 if datasource_name
not in datasource_names
:
126 self
.grafana
.create_datasource(
127 datasource_name
, datasource_type
, datasource_url
130 "Created datasource for k8scluster: %s", k8scluster_id
133 if project
["name"] != "admin":
134 self
.grafana
.create_dashboard(
138 project_name
=project
["name"],
139 datasource_name
=datasource_name
,
142 self
.grafana
.create_dashboard(
146 datasource_name
=datasource_name
,
148 log
.debug("Created dashboard for k8scluster: %s", k8scluster_id
)
151 "Dashboard already exist for k8scluster: %s", k8scluster_id
154 # Reads existing NS list and creates a dashboard for each
155 # TODO lavado: only create for ACTIVE NSRs
156 nsrs
= self
.common_db
.get_nsrs()
159 dashboard_path
= "{}/dashboarder/templates/ns_scoped.json".format(
162 # Collect NS IDs for periodical dashboard clean-up
163 osm_resource_uids
.append(nsr_id
)
164 # Check if the NSR's VNFDs contain metrics
165 # Only one DF at the moment, support for this feature is comming in the future
166 vnfds_profiles
= nsr
["nsd"]["df"][0]["vnf-profile"]
167 for vnf_profile
in vnfds_profiles
:
169 vnfd
= self
.common_db
.get_vnfd_by_id(
170 vnf_profile
["vnfd-id"], create_filter_from_nsr(nsr
)
172 # If there are metrics, create dashboard (if exists)
174 vdu_found
= find_in_list(
176 lambda a_vdu
: "monitoring-parameter" in a_vdu
,
181 if nsr_id
not in dashboard_uids
:
182 nsr_name
= nsr
["name"]
183 project_id
= nsr
["_admin"]["projects_read"][0]
185 # Get project details from commondb
186 project_details
= self
.common_db
.get_project(project_id
)
187 project_name
= project_details
["name"]
188 except Exception as e
:
189 # Project not found in commondb
191 # Serach project in keystone
192 for project
in projects
:
193 if project_id
== project
["_id"]:
194 project_name
= project
["name"]
196 log
.info("Project %s not found", project_id
)
197 log
.debug("Exception %s" % e
)
198 self
.grafana
.create_dashboard(
202 project_name
=project_name
,
204 log
.debug("Created dashboard for NS: %s", nsr_id
)
206 log
.debug("Dashboard already exists")
209 log
.debug("NS does not has metrics")
211 log
.exception("VNFD is not valid or has been renamed")
214 # Delete obsolete dashboards
215 for dashboard_uid
in dashboard_uids
:
216 if dashboard_uid
not in osm_resource_uids
:
217 self
.grafana
.delete_dashboard(dashboard_uid
)
218 log
.debug("Deleted obsolete dashboard: %s", dashboard_uid
)
220 log
.debug("All dashboards in use")
222 # Delete obsolute datasources
223 for datasource_name
in datasource_names
:
224 if datasource_name
not in osm_datasource_names
:
225 self
.grafana
.delete_datasource(datasource_name
)
226 log
.debug("Deleted obsolete datasource: %s", datasource_name
)
228 log
.debug("All dashboards in use")
230 def create_grafana_user(self
, user
):
231 self
.grafana
.create_grafana_users(user
)
233 def delete_non_existing_users(self
):
235 # Get users from keystone
236 users
= self
.keystone
.getUsers()
239 usernames
.append(user
.name
)
240 grafana_users
= self
.grafana
.get_grafana_users()
241 users_to_be_deleted
= list(set(grafana_users
) - set(usernames
))
242 for grafana_user
in users_to_be_deleted
:
243 self
.grafana
.delete_grafana_users(grafana_user
)
245 def create_grafana_team_member(
246 self
, project_data
, userid
=None, project_list
=None, user
=None
252 # Get user details from commondb
253 user
= self
.common_db
.get_user_by_id(userid
)
254 user_name
= user
["username"]
255 except Exception as e
:
256 # User not found in commondb
258 # Search user in keystone
259 user
= self
.keystone
.getUserById(userid
)
260 user_name
= user
.name
262 log
.info("User %s not found", userid
)
263 log
.debug("Exception %s" % e
)
265 # user-project mapping is done by osm cli
266 for proj
in project_data
:
267 project
= self
.common_db
.get_project(proj
["project"])
268 proj_name
= project
["name"]
269 role_obj
= self
.common_db
.get_role_by_id(proj
["role"])
270 is_admin
= role_obj
["permissions"].get("admin")
271 self
.grafana
.create_grafana_teams_members(
272 proj_name
, user_name
, is_admin
, project_list
275 # user-project mapping is done by osm ui
278 users_proj_list
= self
.keystone
.getProjectsById(userid
)
279 for project
in users_proj_list
:
280 proj_list
.append(project
.name
)
282 users_proj_list
= user
.get("project_role_mappings")
283 for project
in users_proj_list
:
284 proj_data
= self
.common_db
.get_project(project
["project"])
285 proj_list
.append(proj_data
["name"])
286 for proj
in project_data
:
288 # Backend authentication type is keystone
290 # Getting project and role objects from keystone using ids
291 role_obj
= self
.keystone
.getRoleById(proj
["role"])
292 proj_data
= self
.keystone
.getProjectsByProjectId(
296 "role object {} {}".format(
297 role_obj
.permissions
, proj_data
.name
300 is_admin
= role_obj
.permissions
["admin"]
302 # Getting project and role objects from keystone using names
303 role_obj
= self
.keystone
.getRoleByName(proj
["role"])[0]
304 proj_data
= self
.keystone
.getProjectsByProjectName(
307 is_admin
= role_obj
.to_dict().get("permissions").get("admin")
309 "role object {} {}".format(
310 role_obj
.to_dict(), proj_data
.name
313 proj_name
= proj_data
.name
315 # Backend authentication type is internal
317 # Getting project and role object from commondb using names
318 role_obj
= self
.common_db
.get_role_by_name(proj
["role"])
319 proj_name
= proj
["project"]
321 # Getting project and role object from commondb using ids
322 role_obj
= self
.common_db
.get_role_by_id(proj
["role"])
323 proj_data
= self
.common_db
.get_project(proj
["project"])
324 proj_name
= proj_data
["name"]
325 is_admin
= role_obj
["permissions"].get("admin")
326 self
.grafana
.create_grafana_teams_members(
327 proj_name
, user_name
, is_admin
, proj_list
330 def create_grafana_team(self
, team_name
):
331 self
.grafana
.create_grafana_teams(team_name
)
333 def delete_grafana_user(self
, user_name
):
334 self
.grafana
.delete_grafana_users(user_name
)
336 def delete_grafana_team(self
, project_name
):
337 self
.grafana
.delete_grafana_team(project_name
)
339 def update_grafana_team(self
, project_new_name
, project_old_name
):
340 self
.grafana
.update_grafana_teams(project_new_name
, project_old_name
)
342 def remove_grafana_team_members(self
, user_id
, proj_data
):
344 # Get user details from commondb
345 user
= self
.common_db
.get_user_by_id(user_id
)
346 user_name
= user
["username"]
347 except Exception as e
:
348 # User not found in commondb
350 # Find user in keystone
351 user
= self
.keystone
.getUserById(user_id
)
352 user_name
= user
.name
354 log
.info("User %s not found", user_id
)
355 log
.debug("Exception %s" % e
)
356 self
.grafana
.remove_grafana_team_member(user_name
, proj_data
)