1 # -*- coding: utf-8 -*-
3 # Copyright 2021 Whitestack, LLC
4 # *************************************************************
6 # This file is part of OSM Monitoring module
7 # All Rights Reserved to Whitestack, LLC
9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
10 # not use this file except in compliance with the License. You may obtain
11 # a copy of the License at
13 # http://www.apache.org/licenses/LICENSE-2.0
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
18 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact: glavado@whitestack.com or fbravo@whitestack.com
25 from osm_mon
.core
.common_db
import CommonDbClient
26 from osm_mon
.core
.config
import Config
27 from osm_mon
.core
.keystone
import KeystoneConnection
28 from osm_mon
.dashboarder
.backends
.grafana
import GrafanaBackend
29 from osm_mon
import __path__
as mon_path
30 from osm_mon
.core
.utils
import find_in_list
, create_filter_from_nsr
33 log
= logging
.getLogger(__name__
)
36 class DashboarderService
:
37 def __init__(self
, config
: Config
):
39 self
.common_db
= CommonDbClient(self
.conf
)
40 self
.grafana
= GrafanaBackend(self
.conf
)
42 if bool(self
.conf
.get("keystone", "enabled")):
43 self
.keystone
= KeystoneConnection(self
.conf
)
47 def create_dashboards(self
):
48 # TODO lavado: migrate these methods to mongo change streams
49 # Lists all dashboards and OSM resources for later comparisons
50 datasource_name_substr
= self
.conf
.get("prometheus-operator", "ds_name_substr")
51 prom_operator_port
= self
.conf
.get("prometheus-operator", "port")
52 dashboard_uids
= self
.grafana
.get_all_dashboard_uids()
53 datasource_names
= self
.grafana
.get_all_datasource_names(datasource_name_substr
)
54 osm_resource_uids
= []
55 osm_datasource_names
= []
58 # Check if keystone is the auth/projects backend and get projects from there
63 lambda project
: {"_id": project
.id, "name": project
.name
},
64 self
.keystone
.getProjects(),
68 log
.error("Cannot retrieve projects from keystone")
70 projects
.extend(self
.common_db
.get_projects())
72 # Reads existing project list and creates a dashboard for each
73 for project
in projects
:
74 project_id
= project
["_id"]
75 # Collect Project IDs for periodical dashboard clean-up
76 osm_resource_uids
.append(project_id
)
77 dashboard_path
= "{}/dashboarder/templates/project_scoped.json".format(
80 cnf_dashboard_path
= "{}/dashboarder/templates/cnf_scoped.json".format(
83 if project_id
not in dashboard_uids
:
84 project_name
= project
["name"]
85 if project_name
!= "admin":
86 # Create project folder in Grafana only if user is not admin.
87 # Admin user's dashboard will be created in default folder
88 self
.grafana
.create_grafana_folders(project_name
)
89 self
.grafana
.create_dashboard(project_id
, project_name
, dashboard_path
)
90 log
.debug("Created dashboard for Project: %s", project_id
)
92 log
.debug("Dashboard already exists")
94 # Read existing k8s cluster list and creates a dashboard for each
95 k8sclusters
= self
.common_db
.get_k8sclusters()
96 for k8scluster
in k8sclusters
:
97 k8scluster_id
= k8scluster
["_id"]
98 k8scluster_name
= k8scluster
["name"]
99 osm_resource_uids
.append(k8scluster_id
)
100 osm_datasource_names
.append("{}-{}".format(datasource_name_substr
, k8scluster_name
))
101 if k8scluster_id
not in dashboard_uids
:
102 projects_read
= k8scluster
["_admin"]["projects_read"]
103 if len(projects_read
) and projects_read
[0] == project_id
:
104 # Collect K8S Cluster IDs for periodical dashboard clean-up
105 k8scluster_address
= k8scluster
["credentials"]["clusters"][0]["cluster"]["server"]
106 # Extract K8S Cluster ip from url
107 k8scluster_ip
= re
.findall(r
'://([\w\-\.]+)', k8scluster_address
)[0]
109 # prometheus-operator url
110 datasource_url
= "http://{}:{}".format(k8scluster_ip
, prom_operator_port
)
112 # Create datsource for prometheus-operator in grafana
113 datasource_type
= "prometheus"
114 datasource_name
= "{}-{}".format(datasource_name_substr
, k8scluster_name
)
115 if datasource_name
not in datasource_names
:
116 self
.grafana
.create_datasource(datasource_name
, datasource_type
, datasource_url
)
117 log
.debug("Created datasource for k8scluster: %s", k8scluster_id
)
119 if project
["name"] != "admin":
120 self
.grafana
.create_dashboard(
121 k8scluster_id
, k8scluster_name
, cnf_dashboard_path
, project_name
=project
["name"],
122 datasource_name
=datasource_name
)
124 self
.grafana
.create_dashboard(
125 k8scluster_id
, k8scluster_name
, cnf_dashboard_path
, datasource_name
=datasource_name
)
126 log
.debug("Created dashboard for k8scluster: %s", k8scluster_id
)
128 log
.debug("Dashboard already exist for k8scluster: %s", k8scluster_id
)
130 # Reads existing NS list and creates a dashboard for each
131 # TODO lavado: only create for ACTIVE NSRs
132 nsrs
= self
.common_db
.get_nsrs()
135 dashboard_path
= "{}/dashboarder/templates/ns_scoped.json".format(
138 # Collect NS IDs for periodical dashboard clean-up
139 osm_resource_uids
.append(nsr_id
)
140 # Check if the NSR's VNFDs contain metrics
141 # Only one DF at the moment, support for this feature is comming in the future
142 vnfds_profiles
= nsr
["nsd"]["df"][0]["vnf-profile"]
143 for vnf_profile
in vnfds_profiles
:
145 vnfd
= self
.common_db
.get_vnfd_by_id(
146 vnf_profile
["vnfd-id"], create_filter_from_nsr(nsr
)
148 # If there are metrics, create dashboard (if exists)
149 vdu_found
= find_in_list(
150 vnfd
["vdu"], lambda a_vdu
: "monitoring-parameter" in a_vdu
153 if nsr_id
not in dashboard_uids
:
154 nsr_name
= nsr
["name"]
155 project_id
= nsr
["_admin"]["projects_read"][0]
157 # Get project details from commondb
158 project_details
= self
.common_db
.get_project(project_id
)
159 project_name
= project_details
["name"]
160 except Exception as e
:
161 # Project not found in commondb
163 # Serach project in keystone
164 for project
in projects
:
165 if project_id
== project
["_id"]:
166 project_name
= project
["name"]
168 log
.info("Project %s not found", project_id
)
169 log
.debug("Exception %s" % e
)
170 self
.grafana
.create_dashboard(
171 nsr_id
, nsr_name
, dashboard_path
, project_name
=project_name
173 log
.debug("Created dashboard for NS: %s", nsr_id
)
175 log
.debug("Dashboard already exists")
178 log
.debug("NS does not has metrics")
180 log
.exception("VNFD is not valid or has been renamed")
183 # Delete obsolete dashboards
184 for dashboard_uid
in dashboard_uids
:
185 if dashboard_uid
not in osm_resource_uids
:
186 self
.grafana
.delete_dashboard(dashboard_uid
)
187 log
.debug("Deleted obsolete dashboard: %s", dashboard_uid
)
189 log
.debug("All dashboards in use")
191 # Delete obsolute datasources
192 for datasource_name
in datasource_names
:
193 if datasource_name
not in osm_datasource_names
:
194 self
.grafana
.delete_datasource(datasource_name
)
195 log
.debug("Deleted obsolete datasource: %s", datasource_name
)
197 log
.debug("All dashboards in use")
199 def create_grafana_user(self
, user
):
200 self
.grafana
.create_grafana_users(user
)
202 def delete_non_existing_users(self
):
204 # Get users from keystone
205 users
= self
.keystone
.getUsers()
208 usernames
.append(user
.name
)
209 grafana_users
= self
.grafana
.get_grafana_users()
210 users_to_be_deleted
= list(set(grafana_users
) - set(usernames
))
211 for grafana_user
in users_to_be_deleted
:
212 self
.grafana
.delete_grafana_users(grafana_user
)
214 def create_grafana_team_member(
215 self
, project_data
, userid
=None, project_list
=None, user
=None
221 # Get user details from commondb
222 user
= self
.common_db
.get_user_by_id(userid
)
223 user_name
= user
["username"]
224 except Exception as e
:
225 # User not found in commondb
227 # Search user in keystone
228 user
= self
.keystone
.getUserById(userid
)
229 user_name
= user
.name
231 log
.info("User %s not found", userid
)
232 log
.debug("Exception %s" % e
)
234 # user-project mapping is done by osm cli
235 for proj
in project_data
:
236 project
= self
.common_db
.get_project(proj
["project"])
237 proj_name
= project
["name"]
238 role_obj
= self
.common_db
.get_role_by_id(proj
["role"])
239 is_admin
= role_obj
["permissions"].get("admin")
240 self
.grafana
.create_grafana_teams_members(
241 proj_name
, user_name
, is_admin
, project_list
244 # user-project mapping is done by osm ui
247 users_proj_list
= self
.keystone
.getProjectsById(userid
)
248 for project
in users_proj_list
:
249 proj_list
.append(project
.name
)
251 users_proj_list
= user
.get("project_role_mappings")
252 for project
in users_proj_list
:
253 proj_data
= self
.common_db
.get_project(project
["project"])
254 proj_list
.append(proj_data
["name"])
255 for proj
in project_data
:
257 # Backend authentication type is keystone
259 # Getting project and role objects from keystone using ids
260 role_obj
= self
.keystone
.getRoleById(proj
["role"])
261 proj_data
= self
.keystone
.getProjectsByProjectId(
265 "role object {} {}".format(
266 role_obj
.permissions
, proj_data
.name
269 is_admin
= role_obj
.permissions
["admin"]
271 # Getting project and role objects from keystone using names
272 role_obj
= self
.keystone
.getRoleByName(proj
["role"])[0]
273 proj_data
= self
.keystone
.getProjectsByProjectName(
276 is_admin
= role_obj
.to_dict().get("permissions").get("admin")
278 "role object {} {}".format(
279 role_obj
.to_dict(), proj_data
.name
282 proj_name
= proj_data
.name
284 # Backend authentication type is internal
286 # Getting project and role object from commondb using names
287 role_obj
= self
.common_db
.get_role_by_name(proj
["role"])
288 proj_name
= proj
["project"]
290 # Getting project and role object from commondb using ids
291 role_obj
= self
.common_db
.get_role_by_id(proj
["role"])
292 proj_data
= self
.common_db
.get_project(proj
["project"])
293 proj_name
= proj_data
["name"]
294 is_admin
= role_obj
["permissions"].get("admin")
295 self
.grafana
.create_grafana_teams_members(
296 proj_name
, user_name
, is_admin
, proj_list
299 def create_grafana_team(self
, team_name
):
300 self
.grafana
.create_grafana_teams(team_name
)
302 def delete_grafana_user(self
, user_name
):
303 self
.grafana
.delete_grafana_users(user_name
)
305 def delete_grafana_team(self
, project_name
):
306 self
.grafana
.delete_grafana_team(project_name
)
308 def update_grafana_team(self
, project_new_name
, project_old_name
):
309 self
.grafana
.update_grafana_teams(project_new_name
, project_old_name
)
311 def remove_grafana_team_members(self
, user_id
, proj_data
):
313 # Get user details from commondb
314 user
= self
.common_db
.get_user_by_id(user_id
)
315 user_name
= user
["username"]
316 except Exception as e
:
317 # User not found in commondb
319 # Find user in keystone
320 user
= self
.keystone
.getUserById(user_id
)
321 user_name
= user
.name
323 log
.info("User %s not found", user_id
)
324 log
.debug("Exception %s" % e
)
325 self
.grafana
.remove_grafana_team_member(user_name
, proj_data
)