1 # -*- coding: utf-8 -*-
3 # Copyright 2021 Whitestack, LLC
4 # *************************************************************
6 # This file is part of OSM Monitoring module
7 # All Rights Reserved to Whitestack, LLC
9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
10 # not use this file except in compliance with the License. You may obtain
11 # a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
18 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact: glavado@whitestack.com or fbravo@whitestack.com
25 from osm_mon
.core
.common_db
import CommonDbClient
26 from osm_mon
.core
.config
import Config
27 from osm_mon
.core
.keystone
import KeystoneConnection
28 from osm_mon
.dashboarder
.backends
.grafana
import GrafanaBackend
29 from osm_mon
import __path__
as mon_path
30 from osm_mon
.core
.utils
import find_in_list
, create_filter_from_nsr
33 log
= logging
.getLogger(__name__
)
36 class DashboarderService
:
37 def __init__(self
, config
: Config
):
39 self
.common_db
= CommonDbClient(self
.conf
)
40 self
.grafana
= GrafanaBackend(self
.conf
)
42 if bool(self
.conf
.get("keystone", "enabled")):
43 self
.keystone
= KeystoneConnection(self
.conf
)
47 def create_dashboards(self
):
48 # TODO lavado: migrate these methods to mongo change streams
49 # Lists all dashboards and OSM resources for later comparisons
50 datasource_name_substr
= self
.conf
.get("prometheus-operator", "ds_name_substr")
51 prom_operator_port
= self
.conf
.get("prometheus-operator", "port")
52 dashboard_uids
= self
.grafana
.get_all_dashboard_uids()
53 datasource_names
= self
.grafana
.get_all_datasource_names(datasource_name_substr
)
54 osm_resource_uids
= []
55 osm_datasource_names
= []
58 # Check if keystone is the auth/projects backend and get projects from there
63 lambda project
: {"_id": project
.id, "name": project
.name
},
64 self
.keystone
.getProjects(),
68 log
.error("Cannot retrieve projects from keystone")
70 projects
.extend(self
.common_db
.get_projects())
72 # Reads existing project list and creates a dashboard for each
73 for project
in projects
:
74 project_id
= project
["_id"]
75 # Collect Project IDs for periodical dashboard clean-up
76 osm_resource_uids
.append(project_id
)
77 dashboard_path
= "{}/dashboarder/templates/project_scoped.json".format(
80 cnf_dashboard_path
= "{}/dashboarder/templates/cnf_scoped.json".format(
83 if project_id
not in dashboard_uids
:
84 project_name
= project
["name"]
85 if project_name
!= "admin":
86 # Create project folder in Grafana only if user is not admin.
87 # Admin user's dashboard will be created in default folder
88 self
.grafana
.create_grafana_folders(project_name
)
89 self
.grafana
.create_dashboard(project_id
, project_name
, dashboard_path
)
90 log
.debug("Created dashboard for Project: %s", project_id
)
92 log
.debug("Dashboard already exists")
94 # Read existing k8s cluster list and creates a dashboard for each
95 k8sclusters
= self
.common_db
.get_k8sclusters()
96 for k8scluster
in k8sclusters
:
97 k8scluster_id
= k8scluster
["_id"]
98 k8scluster_name
= k8scluster
["name"]
99 osm_resource_uids
.append(k8scluster_id
)
100 osm_datasource_names
.append("{}-{}".format(datasource_name_substr
, k8scluster_name
))
101 if k8scluster_id
not in dashboard_uids
:
102 projects_read
= k8scluster
["_admin"]["projects_read"]
103 if len(projects_read
) and projects_read
[0] == project_id
:
104 # Collect K8S Cluster IDs for periodical dashboard clean-up
105 k8scluster_address
= k8scluster
["credentials"]["clusters"][0]["cluster"]["server"]
106 # Extract K8S Cluster ip from url
107 k8scluster_ip
= re
.findall(r
'://([\w\-\.]+)', k8scluster_address
)[0]
109 # prometheus-operator url
110 datasource_url
= "http://{}:{}".format(k8scluster_ip
, prom_operator_port
)
112 # Create datsource for prometheus-operator in grafana
113 datasource_type
= "prometheus"
114 datasource_name
= "{}-{}".format(datasource_name_substr
, k8scluster_name
)
115 if datasource_name
not in datasource_names
:
116 self
.grafana
.create_datasource(datasource_name
, datasource_type
, datasource_url
)
117 log
.debug("Created datasource for k8scluster: %s", k8scluster_id
)
119 if project
["name"] != "admin":
120 self
.grafana
.create_dashboard(
121 k8scluster_id
, k8scluster_name
, cnf_dashboard_path
, project_name
=project
["name"],
122 datasource_name
=datasource_name
)
124 self
.grafana
.create_dashboard(
125 k8scluster_id
, k8scluster_name
, cnf_dashboard_path
, datasource_name
=datasource_name
)
126 log
.debug("Created dashboard for k8scluster: %s", k8scluster_id
)
128 log
.debug("Dashboard already exist for k8scluster: %s", k8scluster_id
)
130 # Reads existing NS list and creates a dashboard for each
131 # TODO lavado: only create for ACTIVE NSRs
132 nsrs
= self
.common_db
.get_nsrs()
135 dashboard_path
= "{}/dashboarder/templates/ns_scoped.json".format(
138 # Collect NS IDs for periodical dashboard clean-up
139 osm_resource_uids
.append(nsr_id
)
140 # Check if the NSR's VNFDs contain metrics
141 # Only one DF at the moment, support for this feature is comming in the future
142 vnfds_profiles
= nsr
["nsd"]["df"][0]["vnf-profile"]
143 for vnf_profile
in vnfds_profiles
:
145 vnfd
= self
.common_db
.get_vnfd_by_id(
146 vnf_profile
["vnfd-id"], create_filter_from_nsr(nsr
)
148 # If there are metrics, create dashboard (if exists)
150 vdu_found
= find_in_list(
151 vnfd
.get("vdu"), lambda a_vdu
: "monitoring-parameter" in a_vdu
156 if nsr_id
not in dashboard_uids
:
157 nsr_name
= nsr
["name"]
158 project_id
= nsr
["_admin"]["projects_read"][0]
160 # Get project details from commondb
161 project_details
= self
.common_db
.get_project(project_id
)
162 project_name
= project_details
["name"]
163 except Exception as e
:
164 # Project not found in commondb
166 # Serach project in keystone
167 for project
in projects
:
168 if project_id
== project
["_id"]:
169 project_name
= project
["name"]
171 log
.info("Project %s not found", project_id
)
172 log
.debug("Exception %s" % e
)
173 self
.grafana
.create_dashboard(
174 nsr_id
, nsr_name
, dashboard_path
, project_name
=project_name
176 log
.debug("Created dashboard for NS: %s", nsr_id
)
178 log
.debug("Dashboard already exists")
181 log
.debug("NS does not has metrics")
183 log
.exception("VNFD is not valid or has been renamed")
186 # Delete obsolete dashboards
187 for dashboard_uid
in dashboard_uids
:
188 if dashboard_uid
not in osm_resource_uids
:
189 self
.grafana
.delete_dashboard(dashboard_uid
)
190 log
.debug("Deleted obsolete dashboard: %s", dashboard_uid
)
192 log
.debug("All dashboards in use")
194 # Delete obsolute datasources
195 for datasource_name
in datasource_names
:
196 if datasource_name
not in osm_datasource_names
:
197 self
.grafana
.delete_datasource(datasource_name
)
198 log
.debug("Deleted obsolete datasource: %s", datasource_name
)
200 log
.debug("All dashboards in use")
202 def create_grafana_user(self
, user
):
203 self
.grafana
.create_grafana_users(user
)
205 def delete_non_existing_users(self
):
207 # Get users from keystone
208 users
= self
.keystone
.getUsers()
211 usernames
.append(user
.name
)
212 grafana_users
= self
.grafana
.get_grafana_users()
213 users_to_be_deleted
= list(set(grafana_users
) - set(usernames
))
214 for grafana_user
in users_to_be_deleted
:
215 self
.grafana
.delete_grafana_users(grafana_user
)
217 def create_grafana_team_member(
218 self
, project_data
, userid
=None, project_list
=None, user
=None
224 # Get user details from commondb
225 user
= self
.common_db
.get_user_by_id(userid
)
226 user_name
= user
["username"]
227 except Exception as e
:
228 # User not found in commondb
230 # Search user in keystone
231 user
= self
.keystone
.getUserById(userid
)
232 user_name
= user
.name
234 log
.info("User %s not found", userid
)
235 log
.debug("Exception %s" % e
)
237 # user-project mapping is done by osm cli
238 for proj
in project_data
:
239 project
= self
.common_db
.get_project(proj
["project"])
240 proj_name
= project
["name"]
241 role_obj
= self
.common_db
.get_role_by_id(proj
["role"])
242 is_admin
= role_obj
["permissions"].get("admin")
243 self
.grafana
.create_grafana_teams_members(
244 proj_name
, user_name
, is_admin
, project_list
247 # user-project mapping is done by osm ui
250 users_proj_list
= self
.keystone
.getProjectsById(userid
)
251 for project
in users_proj_list
:
252 proj_list
.append(project
.name
)
254 users_proj_list
= user
.get("project_role_mappings")
255 for project
in users_proj_list
:
256 proj_data
= self
.common_db
.get_project(project
["project"])
257 proj_list
.append(proj_data
["name"])
258 for proj
in project_data
:
260 # Backend authentication type is keystone
262 # Getting project and role objects from keystone using ids
263 role_obj
= self
.keystone
.getRoleById(proj
["role"])
264 proj_data
= self
.keystone
.getProjectsByProjectId(
268 "role object {} {}".format(
269 role_obj
.permissions
, proj_data
.name
272 is_admin
= role_obj
.permissions
["admin"]
274 # Getting project and role objects from keystone using names
275 role_obj
= self
.keystone
.getRoleByName(proj
["role"])[0]
276 proj_data
= self
.keystone
.getProjectsByProjectName(
279 is_admin
= role_obj
.to_dict().get("permissions").get("admin")
281 "role object {} {}".format(
282 role_obj
.to_dict(), proj_data
.name
285 proj_name
= proj_data
.name
287 # Backend authentication type is internal
289 # Getting project and role object from commondb using names
290 role_obj
= self
.common_db
.get_role_by_name(proj
["role"])
291 proj_name
= proj
["project"]
293 # Getting project and role object from commondb using ids
294 role_obj
= self
.common_db
.get_role_by_id(proj
["role"])
295 proj_data
= self
.common_db
.get_project(proj
["project"])
296 proj_name
= proj_data
["name"]
297 is_admin
= role_obj
["permissions"].get("admin")
298 self
.grafana
.create_grafana_teams_members(
299 proj_name
, user_name
, is_admin
, proj_list
302 def create_grafana_team(self
, team_name
):
303 self
.grafana
.create_grafana_teams(team_name
)
305 def delete_grafana_user(self
, user_name
):
306 self
.grafana
.delete_grafana_users(user_name
)
308 def delete_grafana_team(self
, project_name
):
309 self
.grafana
.delete_grafana_team(project_name
)
311 def update_grafana_team(self
, project_new_name
, project_old_name
):
312 self
.grafana
.update_grafana_teams(project_new_name
, project_old_name
)
314 def remove_grafana_team_members(self
, user_id
, proj_data
):
316 # Get user details from commondb
317 user
= self
.common_db
.get_user_by_id(user_id
)
318 user_name
= user
["username"]
319 except Exception as e
:
320 # User not found in commondb
322 # Find user in keystone
323 user
= self
.keystone
.getUserById(user_id
)
324 user_name
= user
.name
326 log
.info("User %s not found", user_id
)
327 log
.debug("Exception %s" % e
)
328 self
.grafana
.remove_grafana_team_member(user_name
, proj_data
)