osm_mon/dashboarder/service.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2021 Whitestack, LLC
   4 # *************************************************************
   5
   6 # This file is part of OSM Monitoring module
   7 # All Rights Reserved to Whitestack, LLC
   8
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact: glavado@whitestack.com or fbravo@whitestack.com
  22 ##
  23 import logging
  24
  25 from osm_mon.core.common_db import CommonDbClient
  26 from osm_mon.core.config import Config
  27 from osm_mon.core.keystone import KeystoneConnection
  28 from osm_mon.dashboarder.backends.grafana import GrafanaBackend
  29 from osm_mon import __path__ as mon_path
  30 from osm_mon.core.utils import find_in_list, create_filter_from_nsr
  31 import re
  32
  33 log = logging.getLogger(__name__)
  34
  35
  36 class DashboarderService:
  37     def __init__(self, config: Config):
  38         self.conf = config
  39         self.common_db = CommonDbClient(self.conf)
  40         self.grafana = GrafanaBackend(self.conf)
  41
  42         if bool(self.conf.get("keystone", "enabled")):
  43             self.keystone = KeystoneConnection(self.conf)
  44         else:
  45             self.keystone = None
  46
  47     def create_dashboards(self):
  48         # TODO lavado: migrate these methods to mongo change streams
  49         # Lists all dashboards and OSM resources for later comparisons
  50         datasource_name_substr = self.conf.get("prometheus-operator", "ds_name_substr")
  51         prom_operator_port = self.conf.get("prometheus-operator", "port")
  52         dashboard_uids = self.grafana.get_all_dashboard_uids()
  53         datasource_names = self.grafana.get_all_datasource_names(datasource_name_substr)
  54         osm_resource_uids = []
  55         osm_datasource_names = []
  56         projects = []
  57
  58         # Check if keystone is the auth/projects backend and get projects from there
  59         if self.keystone:
  60             try:
  61                 projects.extend(
  62                     map(
  63                         lambda project: {"_id": project.id, "name": project.name},
  64                         self.keystone.getProjects(),
  65                     )
  66                 )
  67             except Exception:
  68                 log.error("Cannot retrieve projects from keystone")
  69         else:
  70             projects.extend(self.common_db.get_projects())
  71
  72         # Reads existing project list and creates a dashboard for each
  73         for project in projects:
  74             project_id = project["_id"]
  75             # Collect Project IDs for periodical dashboard clean-up
  76             osm_resource_uids.append(project_id)
  77             dashboard_path = "{}/dashboarder/templates/project_scoped.json".format(
  78                 mon_path[0]
  79             )
  80             cnf_dashboard_path = "{}/dashboarder/templates/cnf_scoped.json".format(
  81                 mon_path[0]
  82             )
  83             if project_id not in dashboard_uids:
  84                 project_name = project["name"]
  85                 if project_name != "admin":
  86                     # Create project folder in Grafana only if user is not admin.
  87                     # Admin user's dashboard will be created in default folder
  88                     self.grafana.create_grafana_folders(project_name)
  89                 self.grafana.create_dashboard(project_id, project_name, dashboard_path)
  90                 log.debug("Created dashboard for Project: %s", project_id)
  91             else:
  92                 log.debug("Dashboard already exists")
  93
  94             # Read existing k8s cluster list and creates a dashboard for each
  95             k8sclusters = self.common_db.get_k8sclusters()
  96             for k8scluster in k8sclusters:
  97                 k8scluster_id = k8scluster["_id"]
  98                 k8scluster_name = k8scluster["name"]
  99                 osm_resource_uids.append(k8scluster_id)
 100                 osm_datasource_names.append(
 101                     "{}-{}".format(datasource_name_substr, k8scluster_name)
 102                 )
 103                 if k8scluster_id not in dashboard_uids:
 104                     projects_read = k8scluster["_admin"]["projects_read"]
 105                     if len(projects_read) and projects_read[0] == project_id:
 106                         # Collect K8S Cluster IDs for periodical dashboard clean-up
 107                         k8scluster_address = k8scluster["credentials"]["clusters"][0][
 108                             "cluster"
 109                         ]["server"]
 110                         # Extract K8S Cluster ip from url
 111                         k8scluster_ip = re.findall(
 112                             r"://([\w\-\.]+)", k8scluster_address
 113                         )[0]
 114
 115                         # prometheus-operator url
 116                         datasource_url = "http://{}:{}".format(
 117                             k8scluster_ip, prom_operator_port
 118                         )
 119
 120                         # Create datsource for prometheus-operator in grafana
 121                         datasource_type = "prometheus"
 122                         datasource_name = "{}-{}".format(
 123                             datasource_name_substr, k8scluster_name
 124                         )
 125                         if datasource_name not in datasource_names:
 126                             self.grafana.create_datasource(
 127                                 datasource_name, datasource_type, datasource_url
 128                             )
 129                             log.debug(
 130                                 "Created datasource for k8scluster: %s", k8scluster_id
 131                             )
 132
 133                         if project["name"] != "admin":
 134                             self.grafana.create_dashboard(
 135                                 k8scluster_id,
 136                                 k8scluster_name,
 137                                 cnf_dashboard_path,
 138                                 project_name=project["name"],
 139                                 datasource_name=datasource_name,
 140                             )
 141                         else:
 142                             self.grafana.create_dashboard(
 143                                 k8scluster_id,
 144                                 k8scluster_name,
 145                                 cnf_dashboard_path,
 146                                 datasource_name=datasource_name,
 147                             )
 148                         log.debug("Created dashboard for k8scluster: %s", k8scluster_id)
 149                 else:
 150                     log.debug(
 151                         "Dashboard already exist for k8scluster: %s", k8scluster_id
 152                     )
 153
 154         # Reads existing NS list and creates a dashboard for each
 155         # TODO lavado: only create for ACTIVE NSRs
 156         nsrs = self.common_db.get_nsrs()
 157         for nsr in nsrs:
 158             nsr_id = nsr["_id"]
 159             dashboard_path = "{}/dashboarder/templates/ns_scoped.json".format(
 160                 mon_path[0]
 161             )
 162             # Collect NS IDs for periodical dashboard clean-up
 163             osm_resource_uids.append(nsr_id)
 164             # Check if the NSR's VNFDs contain metrics
 165             # Only one DF at the moment, support for this feature is comming in the future
 166             vnfds_profiles = nsr["nsd"]["df"][0]["vnf-profile"]
 167             for vnf_profile in vnfds_profiles:
 168                 try:
 169                     vnfd = self.common_db.get_vnfd_by_id(
 170                         vnf_profile["vnfd-id"], create_filter_from_nsr(nsr)
 171                     )
 172                     # If there are metrics, create dashboard (if exists)
 173                     if vnfd.get("vdu"):
 174                         vdu_found = find_in_list(
 175                             vnfd.get("vdu"),
 176                             lambda a_vdu: "monitoring-parameter" in a_vdu,
 177                         )
 178                     else:
 179                         vdu_found = None
 180                     if vdu_found:
 181                         if nsr_id not in dashboard_uids:
 182                             nsr_name = nsr["name"]
 183                             project_id = nsr["_admin"]["projects_read"][0]
 184                             try:
 185                                 # Get project details from commondb
 186                                 project_details = self.common_db.get_project(project_id)
 187                                 project_name = project_details["name"]
 188                             except Exception as e:
 189                                 # Project not found in commondb
 190                                 if self.keystone:
 191                                     # Serach project in keystone
 192                                     for project in projects:
 193                                         if project_id == project["_id"]:
 194                                             project_name = project["name"]
 195                                 else:
 196                                     log.info("Project %s not found", project_id)
 197                                     log.debug("Exception %s" % e)
 198                             self.grafana.create_dashboard(
 199                                 nsr_id,
 200                                 nsr_name,
 201                                 dashboard_path,
 202                                 project_name=project_name,
 203                             )
 204                             log.debug("Created dashboard for NS: %s", nsr_id)
 205                         else:
 206                             log.debug("Dashboard already exists")
 207                         break
 208                     else:
 209                         log.debug("NS does not has metrics")
 210                 except Exception:
 211                     log.exception("VNFD is not valid or has been renamed")
 212                     continue
 213
 214         # Delete obsolete dashboards
 215         for dashboard_uid in dashboard_uids:
 216             if dashboard_uid not in osm_resource_uids:
 217                 self.grafana.delete_dashboard(dashboard_uid)
 218                 log.debug("Deleted obsolete dashboard: %s", dashboard_uid)
 219             else:
 220                 log.debug("All dashboards in use")
 221
 222         # Delete obsolute datasources
 223         for datasource_name in datasource_names:
 224             if datasource_name not in osm_datasource_names:
 225                 self.grafana.delete_datasource(datasource_name)
 226                 log.debug("Deleted obsolete datasource: %s", datasource_name)
 227             else:
 228                 log.debug("All dashboards in use")
 229
 230     def create_grafana_user(self, user):
 231         self.grafana.create_grafana_users(user)
 232
 233     def delete_non_existing_users(self):
 234         if self.keystone:
 235             # Get users from keystone
 236             users = self.keystone.getUsers()
 237             usernames = []
 238             for user in users:
 239                 usernames.append(user.name)
 240             grafana_users = self.grafana.get_grafana_users()
 241             users_to_be_deleted = list(set(grafana_users) - set(usernames))
 242             for grafana_user in users_to_be_deleted:
 243                 self.grafana.delete_grafana_users(grafana_user)
 244
 245     def create_grafana_team_member(
 246         self, project_data, userid=None, project_list=None, user=None
 247     ):
 248         if user:
 249             user_name = user
 250         else:
 251             try:
 252                 # Get user details from  commondb
 253                 user = self.common_db.get_user_by_id(userid)
 254                 user_name = user["username"]
 255             except Exception as e:
 256                 # User not found in commondb
 257                 if self.keystone:
 258                     # Search user in keystone
 259                     user = self.keystone.getUserById(userid)
 260                     user_name = user.name
 261                 else:
 262                     log.info("User %s not found", userid)
 263                     log.debug("Exception %s" % e)
 264         if project_list:
 265             # user-project mapping is done by osm cli
 266             for proj in project_data:
 267                 project = self.common_db.get_project(proj["project"])
 268                 proj_name = project["name"]
 269                 role_obj = self.common_db.get_role_by_id(proj["role"])
 270                 is_admin = role_obj["permissions"].get("admin")
 271                 self.grafana.create_grafana_teams_members(
 272                     proj_name, user_name, is_admin, project_list
 273                 )
 274         else:
 275             # user-project mapping is done by osm ui
 276             proj_list = []
 277             if self.keystone:
 278                 users_proj_list = self.keystone.getProjectsById(userid)
 279                 for project in users_proj_list:
 280                     proj_list.append(project.name)
 281             else:
 282                 users_proj_list = user.get("project_role_mappings")
 283                 for project in users_proj_list:
 284                     proj_data = self.common_db.get_project(project["project"])
 285                     proj_list.append(proj_data["name"])
 286             for proj in project_data:
 287                 if self.keystone:
 288                     # Backend authentication type is keystone
 289                     try:
 290                         # Getting project and role objects from keystone using ids
 291                         role_obj = self.keystone.getRoleById(proj["role"])
 292                         proj_data = self.keystone.getProjectsByProjectId(
 293                             proj["project"]
 294                         )
 295                         log.info(
 296                             "role object {} {}".format(
 297                                 role_obj.permissions, proj_data.name
 298                             )
 299                         )
 300                         is_admin = role_obj.permissions["admin"]
 301                     except Exception:
 302                         # Getting project and role objects from keystone using names
 303                         role_obj = self.keystone.getRoleByName(proj["role"])[0]
 304                         proj_data = self.keystone.getProjectsByProjectName(
 305                             proj["project"]
 306                         )[0]
 307                         is_admin = role_obj.to_dict().get("permissions").get("admin")
 308                         log.info(
 309                             "role object {} {}".format(
 310                                 role_obj.to_dict(), proj_data.name
 311                             )
 312                         )
 313                     proj_name = proj_data.name
 314                 else:
 315                     # Backend authentication type is internal
 316                     try:
 317                         # Getting project and role object from commondb using names
 318                         role_obj = self.common_db.get_role_by_name(proj["role"])
 319                         proj_name = proj["project"]
 320                     except Exception:
 321                         # Getting project and role object from commondb using ids
 322                         role_obj = self.common_db.get_role_by_id(proj["role"])
 323                         proj_data = self.common_db.get_project(proj["project"])
 324                         proj_name = proj_data["name"]
 325                     is_admin = role_obj["permissions"].get("admin")
 326                 self.grafana.create_grafana_teams_members(
 327                     proj_name, user_name, is_admin, proj_list
 328                 )
 329
 330     def create_grafana_team(self, team_name):
 331         self.grafana.create_grafana_teams(team_name)
 332
 333     def delete_grafana_user(self, user_name):
 334         self.grafana.delete_grafana_users(user_name)
 335
 336     def delete_grafana_team(self, project_name):
 337         self.grafana.delete_grafana_team(project_name)
 338
 339     def update_grafana_team(self, project_new_name, project_old_name):
 340         self.grafana.update_grafana_teams(project_new_name, project_old_name)
 341
 342     def remove_grafana_team_members(self, user_id, proj_data):
 343         try:
 344             # Get user details from  commondb
 345             user = self.common_db.get_user_by_id(user_id)
 346             user_name = user["username"]
 347         except Exception as e:
 348             # User not found in commondb
 349             if self.keystone:
 350                 # Find user in keystone
 351                 user = self.keystone.getUserById(user_id)
 352                 user_name = user.name
 353             else:
 354                 log.info("User %s not found", user_id)
 355                 log.debug("Exception %s" % e)
 356         self.grafana.remove_grafana_team_member(user_name, proj_data)