osm_mon/dashboarder/service.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2021 Whitestack, LLC
   4 # *************************************************************
   5
   6 # This file is part of OSM Monitoring module
   7 # All Rights Reserved to Whitestack, LLC
   8
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact: glavado@whitestack.com or fbravo@whitestack.com
  22 ##
  23 import logging
  24
  25 from osm_mon.core.common_db import CommonDbClient
  26 from osm_mon.core.config import Config
  27 from osm_mon.core.keystone import KeystoneConnection
  28 from osm_mon.dashboarder.backends.grafana import GrafanaBackend
  29 from osm_mon import __path__ as mon_path
  30 from osm_mon.core.utils import find_in_list, create_filter_from_nsr
  31 import re
  32
  33 log = logging.getLogger(__name__)
  34
  35
  36 class DashboarderService:
  37     def __init__(self, config: Config):
  38         self.conf = config
  39         self.common_db = CommonDbClient(self.conf)
  40         self.grafana = GrafanaBackend(self.conf)
  41
  42         if bool(self.conf.get("keystone", "enabled")):
  43             self.keystone = KeystoneConnection(self.conf)
  44         else:
  45             self.keystone = None
  46
  47     def create_dashboards(self):
  48         # TODO lavado: migrate these methods to mongo change streams
  49         # Lists all dashboards and OSM resources for later comparisons
  50         datasource_name_substr = self.conf.get("prometheus-operator", "ds_name_substr")
  51         prom_operator_port = self.conf.get("prometheus-operator", "port")
  52         dashboard_uids = self.grafana.get_all_dashboard_uids()
  53         datasource_names = self.grafana.get_all_datasource_names(datasource_name_substr)
  54         osm_resource_uids = []
  55         osm_datasource_names = []
  56         projects = []
  57
  58         # Check if keystone is the auth/projects backend and get projects from there
  59         if self.keystone:
  60             try:
  61                 projects.extend(
  62                     map(
  63                         lambda project: {"_id": project.id, "name": project.name},
  64                         self.keystone.getProjects(),
  65                     )
  66                 )
  67             except Exception:
  68                 log.error("Cannot retrieve projects from keystone")
  69         else:
  70             projects.extend(self.common_db.get_projects())
  71
  72         # Reads existing project list and creates a dashboard for each
  73         for project in projects:
  74             project_id = project["_id"]
  75             # Collect Project IDs for periodical dashboard clean-up
  76             osm_resource_uids.append(project_id)
  77             dashboard_path = "{}/dashboarder/templates/project_scoped.json".format(
  78                 mon_path[0]
  79             )
  80             cnf_dashboard_path = "{}/dashboarder/templates/cnf_scoped.json".format(
  81                 mon_path[0]
  82             )
  83             if project_id not in dashboard_uids:
  84                 project_name = project["name"]
  85                 if project_name != "admin":
  86                     # Create project folder in Grafana only if user is not admin.
  87                     # Admin user's dashboard will be created in default folder
  88                     self.grafana.create_grafana_folders(project_name)
  89                 self.grafana.create_dashboard(project_id, project_name, dashboard_path)
  90                 log.debug("Created dashboard for Project: %s", project_id)
  91             else:
  92                 log.debug("Dashboard already exists")
  93
  94             # Read existing k8s cluster list and creates a dashboard for each
  95             k8sclusters = self.common_db.get_k8sclusters()
  96             for k8scluster in k8sclusters:
  97                 k8scluster_id = k8scluster["_id"]
  98                 k8scluster_name = k8scluster["name"]
  99                 osm_resource_uids.append(k8scluster_id)
 100                 osm_datasource_names.append("{}-{}".format(datasource_name_substr, k8scluster_name))
 101                 if k8scluster_id not in dashboard_uids:
 102                     projects_read = k8scluster["_admin"]["projects_read"]
 103                     if len(projects_read) and projects_read[0] == project_id:
 104                         # Collect K8S Cluster IDs for periodical dashboard clean-up
 105                         k8scluster_address = k8scluster["credentials"]["clusters"][0]["cluster"]["server"]
 106                         # Extract K8S Cluster ip from url
 107                         k8scluster_ip = re.findall(r'://([\w\-\.]+)', k8scluster_address)[0]
 108
 109                         # prometheus-operator url
 110                         datasource_url = "http://{}:{}".format(k8scluster_ip, prom_operator_port)
 111
 112                         # Create datsource for prometheus-operator in grafana
 113                         datasource_type = "prometheus"
 114                         datasource_name = "{}-{}".format(datasource_name_substr, k8scluster_name)
 115                         if datasource_name not in datasource_names:
 116                             self.grafana.create_datasource(datasource_name, datasource_type, datasource_url)
 117                             log.debug("Created datasource for k8scluster: %s", k8scluster_id)
 118
 119                         if project["name"] != "admin":
 120                             self.grafana.create_dashboard(
 121                                 k8scluster_id, k8scluster_name, cnf_dashboard_path, project_name=project["name"],
 122                                 datasource_name=datasource_name)
 123                         else:
 124                             self.grafana.create_dashboard(
 125                                 k8scluster_id, k8scluster_name, cnf_dashboard_path, datasource_name=datasource_name)
 126                         log.debug("Created dashboard for k8scluster: %s", k8scluster_id)
 127                 else:
 128                     log.debug("Dashboard already exist for k8scluster: %s", k8scluster_id)
 129
 130         # Reads existing NS list and creates a dashboard for each
 131         # TODO lavado: only create for ACTIVE NSRs
 132         nsrs = self.common_db.get_nsrs()
 133         for nsr in nsrs:
 134             nsr_id = nsr["_id"]
 135             dashboard_path = "{}/dashboarder/templates/ns_scoped.json".format(
 136                 mon_path[0]
 137             )
 138             # Collect NS IDs for periodical dashboard clean-up
 139             osm_resource_uids.append(nsr_id)
 140             # Check if the NSR's VNFDs contain metrics
 141             # Only one DF at the moment, support for this feature is comming in the future
 142             vnfds_profiles = nsr["nsd"]["df"][0]["vnf-profile"]
 143             for vnf_profile in vnfds_profiles:
 144                 try:
 145                     vnfd = self.common_db.get_vnfd_by_id(
 146                         vnf_profile["vnfd-id"], create_filter_from_nsr(nsr)
 147                     )
 148                     # If there are metrics, create dashboard (if exists)
 149                     if vnfd.get("vdu"):
 150                         vdu_found = find_in_list(
 151                             vnfd.get("vdu"), lambda a_vdu: "monitoring-parameter" in a_vdu
 152                         )
 153                     else:
 154                         vdu_found = None
 155                     if vdu_found:
 156                         if nsr_id not in dashboard_uids:
 157                             nsr_name = nsr["name"]
 158                             project_id = nsr["_admin"]["projects_read"][0]
 159                             try:
 160                                 # Get project details from commondb
 161                                 project_details = self.common_db.get_project(project_id)
 162                                 project_name = project_details["name"]
 163                             except Exception as e:
 164                                 # Project not found in commondb
 165                                 if self.keystone:
 166                                     # Serach project in keystone
 167                                     for project in projects:
 168                                         if project_id == project["_id"]:
 169                                             project_name = project["name"]
 170                                 else:
 171                                     log.info("Project %s not found", project_id)
 172                                     log.debug("Exception %s" % e)
 173                             self.grafana.create_dashboard(
 174                                 nsr_id, nsr_name, dashboard_path, project_name=project_name
 175                             )
 176                             log.debug("Created dashboard for NS: %s", nsr_id)
 177                         else:
 178                             log.debug("Dashboard already exists")
 179                         break
 180                     else:
 181                         log.debug("NS does not has metrics")
 182                 except Exception:
 183                     log.exception("VNFD is not valid or has been renamed")
 184                     continue
 185
 186         # Delete obsolete dashboards
 187         for dashboard_uid in dashboard_uids:
 188             if dashboard_uid not in osm_resource_uids:
 189                 self.grafana.delete_dashboard(dashboard_uid)
 190                 log.debug("Deleted obsolete dashboard: %s", dashboard_uid)
 191             else:
 192                 log.debug("All dashboards in use")
 193
 194         # Delete obsolute datasources
 195         for datasource_name in datasource_names:
 196             if datasource_name not in osm_datasource_names:
 197                 self.grafana.delete_datasource(datasource_name)
 198                 log.debug("Deleted obsolete datasource: %s", datasource_name)
 199             else:
 200                 log.debug("All dashboards in use")
 201
 202     def create_grafana_user(self, user):
 203         self.grafana.create_grafana_users(user)
 204
 205     def delete_non_existing_users(self):
 206         if self.keystone:
 207             # Get users from keystone
 208             users = self.keystone.getUsers()
 209             usernames = []
 210             for user in users:
 211                 usernames.append(user.name)
 212             grafana_users = self.grafana.get_grafana_users()
 213             users_to_be_deleted = list(set(grafana_users) - set(usernames))
 214             for grafana_user in users_to_be_deleted:
 215                 self.grafana.delete_grafana_users(grafana_user)
 216
 217     def create_grafana_team_member(
 218         self, project_data, userid=None, project_list=None, user=None
 219     ):
 220         if user:
 221             user_name = user
 222         else:
 223             try:
 224                 # Get user details from  commondb
 225                 user = self.common_db.get_user_by_id(userid)
 226                 user_name = user["username"]
 227             except Exception as e:
 228                 # User not found in commondb
 229                 if self.keystone:
 230                     # Search user in keystone
 231                     user = self.keystone.getUserById(userid)
 232                     user_name = user.name
 233                 else:
 234                     log.info("User %s not found", userid)
 235                     log.debug("Exception %s" % e)
 236         if project_list:
 237             # user-project mapping is done by osm cli
 238             for proj in project_data:
 239                 project = self.common_db.get_project(proj["project"])
 240                 proj_name = project["name"]
 241                 role_obj = self.common_db.get_role_by_id(proj["role"])
 242                 is_admin = role_obj["permissions"].get("admin")
 243                 self.grafana.create_grafana_teams_members(
 244                     proj_name, user_name, is_admin, project_list
 245                 )
 246         else:
 247             # user-project mapping is done by osm ui
 248             proj_list = []
 249             if self.keystone:
 250                 users_proj_list = self.keystone.getProjectsById(userid)
 251                 for project in users_proj_list:
 252                     proj_list.append(project.name)
 253             else:
 254                 users_proj_list = user.get("project_role_mappings")
 255                 for project in users_proj_list:
 256                     proj_data = self.common_db.get_project(project["project"])
 257                     proj_list.append(proj_data["name"])
 258             for proj in project_data:
 259                 if self.keystone:
 260                     # Backend authentication type is keystone
 261                     try:
 262                         # Getting project and role objects from keystone using ids
 263                         role_obj = self.keystone.getRoleById(proj["role"])
 264                         proj_data = self.keystone.getProjectsByProjectId(
 265                             proj["project"]
 266                         )
 267                         log.info(
 268                             "role object {} {}".format(
 269                                 role_obj.permissions, proj_data.name
 270                             )
 271                         )
 272                         is_admin = role_obj.permissions["admin"]
 273                     except Exception:
 274                         # Getting project and role objects from keystone using names
 275                         role_obj = self.keystone.getRoleByName(proj["role"])[0]
 276                         proj_data = self.keystone.getProjectsByProjectName(
 277                             proj["project"]
 278                         )[0]
 279                         is_admin = role_obj.to_dict().get("permissions").get("admin")
 280                         log.info(
 281                             "role object {} {}".format(
 282                                 role_obj.to_dict(), proj_data.name
 283                             )
 284                         )
 285                     proj_name = proj_data.name
 286                 else:
 287                     # Backend authentication type is internal
 288                     try:
 289                         # Getting project and role object from commondb using names
 290                         role_obj = self.common_db.get_role_by_name(proj["role"])
 291                         proj_name = proj["project"]
 292                     except Exception:
 293                         # Getting project and role object from commondb using ids
 294                         role_obj = self.common_db.get_role_by_id(proj["role"])
 295                         proj_data = self.common_db.get_project(proj["project"])
 296                         proj_name = proj_data["name"]
 297                     is_admin = role_obj["permissions"].get("admin")
 298                 self.grafana.create_grafana_teams_members(
 299                     proj_name, user_name, is_admin, proj_list
 300                 )
 301
 302     def create_grafana_team(self, team_name):
 303         self.grafana.create_grafana_teams(team_name)
 304
 305     def delete_grafana_user(self, user_name):
 306         self.grafana.delete_grafana_users(user_name)
 307
 308     def delete_grafana_team(self, project_name):
 309         self.grafana.delete_grafana_team(project_name)
 310
 311     def update_grafana_team(self, project_new_name, project_old_name):
 312         self.grafana.update_grafana_teams(project_new_name, project_old_name)
 313
 314     def remove_grafana_team_members(self, user_id, proj_data):
 315         try:
 316             # Get user details from  commondb
 317             user = self.common_db.get_user_by_id(user_id)
 318             user_name = user["username"]
 319         except Exception as e:
 320             # User not found in commondb
 321             if self.keystone:
 322                 # Find user in keystone
 323                 user = self.keystone.getUserById(user_id)
 324                 user_name = user.name
 325             else:
 326                 log.info("User %s not found", user_id)
 327                 log.debug("Exception %s" % e)
 328         self.grafana.remove_grafana_team_member(user_name, proj_data)