MON changes for Grafana Multitenancy feature.

Change-Id: Iff425db1236683d8bf8bcf4ec4dac0c23484cdfa
Signed-off-by: agarwalat <atul.agarwal@altran.com>

diff --git a/osm_mon/cmd/mon_dashboarder.py b/osm_mon/cmd/mon_dashboarder.py
index 211038d..b49b570 100644 (file)
--- a/osm_mon/cmd/mon_dashboarder.py
+++ b/osm_mon/cmd/mon_dashboarder.py
@@ -27,6 +27,7 @@ import sys
 
 from osm_mon.core.config import Config
 from osm_mon.dashboarder.dashboarder import Dashboarder
 
 from osm_mon.core.config import Config
 from osm_mon.dashboarder.dashboarder import Dashboarder

+import threading

 
 
 def main():
 
 
 def main():


@@ -47,6 +48,9 @@ def main():
     log.info("Starting MON Dashboarder...")
     log.debug("Config: %s", cfg.conf)
     dashboarder = Dashboarder(cfg)
     log.info("Starting MON Dashboarder...")
     log.debug("Config: %s", cfg.conf)
     dashboarder = Dashboarder(cfg)

+    log.info("Starting MON kafka Consumer...")
+    threading.Thread(target=dashboarder.run, args=()).start()
+    log.info("Starting MON Dashboarder...")

     dashboarder.dashboard_forever()
 
 
     dashboarder.dashboard_forever()
 
 

diff --git a/osm_mon/core/common_db.py b/osm_mon/core/common_db.py
index 983d84d..e43c6cd 100644 (file)
--- a/osm_mon/core/common_db.py
+++ b/osm_mon/core/common_db.py
@@ -153,3 +153,12 @@ class CommonDbClient:
         for alarm_dict in alarm_dicts:
             alarms.append(Alarm.from_dict(alarm_dict))
         return alarms
         for alarm_dict in alarm_dicts:
             alarms.append(Alarm.from_dict(alarm_dict))
         return alarms

+
+    def get_user(self, username: str):
+        return self.common_db.get_one('users', {'username': username})
+
+    def get_user_by_id(self, userid: str):
+        return self.common_db.get_one('users', {'_id': userid})
+
+    def get_role_by_name(self, name: str):
+        return self.common_db.get_one('roles', {'name': name})

diff --git a/osm_mon/dashboarder/backends/grafana.py b/osm_mon/dashboarder/backends/grafana.py
index acc602e..6fb6b94 100644 (file)
--- a/osm_mon/dashboarder/backends/grafana.py
+++ b/osm_mon/dashboarder/backends/grafana.py
@@ -1,72 +1,186 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2018 Whitestack, LLC
-# *************************************************************
-
-# This file is part of OSM Monitoring module
-# All Rights Reserved to Whitestack, LLC
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-
-#         http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact: glavado@whitestack.com or fbravo@whitestack.com
-##
-import logging
-import requests
-import base64
-from osm_mon.core.config import Config
-
-log = logging.getLogger(__name__)
-
-
-class GrafanaBackend:
-    def __init__(self, config: Config):
-        self.conf = config
-        self.url = config.get('grafana', 'url')
-        grafana_user = config.get("grafana", "user")
-        grafana_password = config.get("grafana", "password")
-        self.headers = {
-            'content-type': "application/json",
-            'authorization': "Basic %s" % base64.b64encode(
-                (grafana_user + ":" + grafana_password).encode("utf-8")).decode()
-        }
-
-    def get_all_dashboard_uids(self):
-        # Gets only dashboards that were automated by OSM (with tag 'osm_automated')
-        response = requests.request("GET", self.url + "/api/search?tag=osm_automated", headers=self.headers)
-        dashboards = response.json()
-        dashboard_uids = []
-        for dashboard in dashboards:
-            dashboard_uids.append(dashboard['uid'])
-        log.debug("Searching for all dashboard uids: %s", dashboard_uids)
-        return dashboard_uids
-
-    def get_dashboard_status(self, uid):
-        response = requests.request("GET", self.url + "/api/dashboards/uid/" + uid, headers=self.headers)
-        log.debug("Searching for dashboard result: %s", response.text)
-        return response
-
-    def create_dashboard(self, uid, name, json_file):
-        with open(json_file) as f:
-            dashboard_data = f.read()
-
-        dashboard_data = dashboard_data.replace('OSM_ID', uid).replace('OSM_NAME', name)
-
-        response = requests.request(
-            "POST", self.url + "/api/dashboards/db/",  data=dashboard_data, headers=self.headers)
-        log.debug("Creating dashboard result: %s", response.text)
-        return response
-
-    def delete_dashboard(self, uid):
-        response = requests.request("DELETE", self.url + "/api/dashboards/uid/" + uid, headers=self.headers)
-        log.debug("Delete dashboard result: %s", response.text)
-        return response
+# -*- coding: utf-8 -*-\r
+\r
+# Copyright 2018 Whitestack, LLC\r
+# *************************************************************\r
+\r
+# This file is part of OSM Monitoring module\r
+# All Rights Reserved to Whitestack, LLC\r
+\r
+# Licensed under the Apache License, Version 2.0 (the "License"); you may\r
+# not use this file except in compliance with the License. You may obtain\r
+# a copy of the License at\r
+\r
+#         http://www.apache.org/licenses/LICENSE-2.0\r
+\r
+# Unless required by applicable law or agreed to in writing, software\r
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT\r
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\r
+# License for the specific language governing permissions and limitations\r
+# under the License.\r
+# For those usages not covered by the Apache License, Version 2.0 please\r
+# contact: glavado@whitestack.com or fbravo@whitestack.com\r
+##\r
+import logging\r
+import requests\r
+import base64\r
+import json\r
+from osm_mon.core.config import Config\r
+\r
+log = logging.getLogger(__name__)\r
+\r
+\r
+class GrafanaBackend:\r
+    def __init__(self, config: Config):\r
+        self.conf = config\r
+        self.url = config.get('grafana', 'url')\r
+        grafana_user = config.get("grafana", "user")\r
+        grafana_password = config.get("grafana", "password")\r
+        self.headers = {\r
+            'content-type': "application/json",\r
+            'authorization': "Basic %s" % base64.b64encode(\r
+                (grafana_user + ":" + grafana_password).encode("utf-8")).decode()\r
+        }\r
+\r
+    def get_all_dashboard_uids(self):\r
+        # Gets only dashboards that were automated by OSM (with tag 'osm_automated')\r
+        response = requests.request("GET", self.url + "/api/search?tag=osm_automated", headers=self.headers)\r
+        dashboards = response.json()\r
+        dashboard_uids = []\r
+        for dashboard in dashboards:\r
+            dashboard_uids.append(dashboard['uid'])\r
+        log.debug("Searching for all dashboard uids: %s", dashboard_uids)\r
+        return dashboard_uids\r
+\r
+    def get_dashboard_status(self, uid):\r
+        response = requests.request("GET", self.url + "/api/dashboards/uid/" + uid, headers=self.headers)\r
+        log.debug("Searching for dashboard result: %s", response.text)\r
+        return response\r
+\r
+    def create_dashboard(self, uid, name, json_file, project_name=None):\r
+        try:\r
+            with open(json_file) as f:\r
+                dashboard_data = f.read()\r
+\r
+            dashboard_data = dashboard_data.replace('OSM_ID', uid).replace('OSM_NAME', name)\r
+\r
+            response = requests.request(\r
+                "POST", self.url + "/api/dashboards/db/", data=dashboard_data, headers=self.headers)\r
+            # get team id\r
+            if project_name is not None:\r
+                name = project_name\r
+            response_team = requests.request(\r
+                "GET", self.url + "/api/teams/search?name={}".format(name), headers=self.headers)\r
+            if len(json.loads(response_team.text)["teams"]) > 0:\r
+                team_id = json.loads(response_team.text)["teams"][0]["id"]\r
+                permission_data = {"items": [{"teamId": team_id, "permission": 2}, ]}\r
+                # provide permission to dashboard\r
+                dahboard_id = json.loads(response.text)["id"]\r
+                response = requests.request(\r
+                    "POST", self.url + "/api/dashboards/id/{}/permissions".format(dahboard_id), json=permission_data,\r
+                    headers=self.headers)\r
+            return response\r
+        except Exception:\r
+            log.exception("Exception processing message: ")\r
+\r
+    def delete_dashboard(self, uid):\r
+        response = requests.request("DELETE", self.url + "/api/dashboards/uid/" + uid, headers=self.headers)\r
+        log.debug("Delete dashboard result: %s", response.text)\r
+        return response\r
+\r
+    def create_grafana_users(self, user):\r
+        email = "{}@osm.etsi.org".format(user)\r
+        user_payload = {\r
+            "name": user,\r
+            "email": email,\r
+            "login": user,\r
+            "password": user,\r
+        }\r
+        response_users = requests.request("POST", self.url + "/api/admin/users/", json=user_payload,\r
+                                          headers=self.headers)\r
+        json_data = json.loads(response_users.text)\r
+        url = "/api/org/users/{}/".format(json_data["id"])\r
+        permission_payload = {"role": "Editor", }\r
+        requests.request("PATCH", self.url + url, json=permission_payload, headers=self.headers)\r
+        log.info("Grafana user created: %s", response_users.text)\r
+        return response_users\r
+\r
+    # create grafana team with member\r
+    def create_grafana_teams_members(self, project_name, user_name, is_admin, proj_list):\r
+        # check if user exist in grafana or not\r
+        user_response = requests.request("GET", self.url + "/api/users/lookup?loginOrEmail={}".format(user_name),\r
+                                         headers=self.headers)\r
+        user_obj = json.loads(user_response.text)\r
+        if user_response.status_code != 200:\r
+            user_response = self.create_grafana_users(user_name)\r
+            user_obj = json.loads(user_response.text)\r
+\r
+        user_id = user_obj["id"]\r
+\r
+        # Get Teams for user\r
+        team_objs = requests.request("GET", self.url + "/api/users/{}/teams".format(user_id), headers=self.headers)\r
+        team_obj = json.loads(team_objs.text)\r
+        team_list = []\r
+        if len(team_obj):\r
+            for team in team_obj:\r
+                team_list.append(team["name"])\r
+\r
+        proj_unlink = set(team_list) - set(proj_list)\r
+        for prj in proj_unlink:\r
+            response_team = requests.request("GET", self.url + "/api/teams/search?name={}".format(prj),\r
+                                             headers=self.headers)\r
+            team_id = json.loads(response_team.text)["teams"][0]["id"]\r
+            requests.request("DELETE", self.url + "/api/teams/{}/members/{}".format(team_id, user_id),\r
+                             headers=self.headers)\r
+        # add member to team\r
+        response_team = requests.request("GET", self.url + "/api/teams/search?name={}".format(project_name),\r
+                                         headers=self.headers)\r
+        team_id = json.loads(response_team.text)["teams"][0]["id"]\r
+        if project_name not in team_list:\r
+            member_payload = {\r
+                "userId": user_id\r
+            }\r
+            requests.request("POST", self.url + "/api/teams/{}/members".format(team_id), json=member_payload,\r
+                             headers=self.headers)\r
+        # if role is admin change permission to admin\r
+        if is_admin:\r
+            url = "/api/org/users/{}/".format(user_id)\r
+            permission_payload = {"role": "Admin", }\r
+            requests.request("PATCH", self.url + url, json=permission_payload, headers=self.headers)\r
+        log.info("Member added to grafana team")\r
+        return response_team\r
+\r
+    # create grafana team\r
+    def create_grafana_teams(self, team_name):\r
+        team_payload = {"name": team_name, }\r
+        response = requests.request("POST", self.url + "/api/teams", json=team_payload, headers=self.headers)\r
+        log.info("Grafana Team is created: %s", response.text)\r
+\r
+    def delete_grafana_users(self, user_name):\r
+        # find user id\r
+        response_id = requests.request("GET", self.url + "/api/users/lookup?loginOrEmail={}".format(user_name),\r
+                                       headers=self.headers)\r
+        try:\r
+            user_id = json.loads(response_id.text)["id"]\r
+        except Exception:\r
+            log.exception("Exception processing message: ")\r
+        # delete user\r
+        response = requests.request("DELETE", self.url + "/api/admin/users/{}".format(user_id), headers=self.headers)\r
+        log.info("Grafana user deleted: %s", response.text)\r
+        return response\r
+\r
+    def delete_grafana_team(self, project_name):\r
+        team_obj = requests.request("GET", self.url + "/api/teams/search?name={}".format(project_name),\r
+                                    headers=self.headers)\r
+        team_id = json.loads(team_obj.text)["teams"][0]["id"]\r
+        response = requests.request("DELETE", self.url + "/api/teams/{}".format(team_id), headers=self.headers)\r
+        log.info("Grafana team deleated: %s", response.text)\r
+        return response\r
+\r
+    def update_grafana_teams(self, project_new_name, project_old_name):\r
+        team_obj = requests.request("GET", self.url + "/api/teams/search?name={}".format(project_old_name),\r
+                                    headers=self.headers)\r
+        team_id = json.loads(team_obj.text)["teams"][0]["id"]\r
+        data = {"name": project_new_name, }\r
+        response = requests.request("PUT", self.url + "/api/teams/{}".format(team_id), json=data, headers=self.headers)\r
+        log.info("Grafana team updated %s", response.text)\r
+        return response\r

diff --git a/osm_mon/dashboarder/dashboarder.py b/osm_mon/dashboarder/dashboarder.py
index bb2935a..30f27d1 100644 (file)
--- a/osm_mon/dashboarder/dashboarder.py
+++ b/osm_mon/dashboarder/dashboarder.py
@@ -24,17 +24,64 @@ import logging
 import time
 import socket
 import peewee
 import time
 import socket
 import peewee

+import asyncio

 
 from osm_mon.dashboarder.service import DashboarderService
 from osm_mon.core.config import Config
 
 from osm_mon.dashboarder.service import DashboarderService
 from osm_mon.core.config import Config

+from osm_mon.core.message_bus_client import MessageBusClient

 
 log = logging.getLogger(__name__)
 
 
 class Dashboarder:
 
 log = logging.getLogger(__name__)
 
 
 class Dashboarder:

-    def __init__(self, config: Config):
+    def __init__(self, config: Config, loop=None):

         self.conf = config
         self.service = DashboarderService(config)
         self.conf = config
         self.service = DashboarderService(config)

+        if not loop:
+            loop = asyncio.get_event_loop()
+        self.loop = loop
+        self.msg_bus = MessageBusClient(config)
+
+    # run consumer for grafana user management
+    def run(self):
+        self.loop.run_until_complete(self.start())
+
+    async def start(self):
+        topics = ["users", "project"]
+        await self.msg_bus.aioread(topics, self._user_msg)
+
+    async def _user_msg(self, topic, key, values):
+        log.debug("Message from kafka bus received: topic: %s and values: %s and key: %s", topic, values, key)
+        try:
+            if topic == "users" and key == "created":
+                log.debug("Received message from kafka for creating user")
+                user = values['username']
+                self.service.create_grafana_user(user)
+            elif topic == "users" and key == "deleted":
+                log.debug("Received message from kafka for deleting user")
+                user = values['username']
+                self.service.delete_grafana_user(user)
+                log.info("Grafana user deleted: %s", user)
+            elif topic == "users" and key == "edited":
+                log.debug("Received message from kafka for associating user to team")
+                user_id = values["_id"]
+                project_data = values["changes"]["project_role_mappings"]
+                self.service.create_grafana_team_member(project_data, user_id)
+            elif topic == "project" and key == "created":
+                log.debug("Received message from kafka for creating team")
+                team_name = values["name"]
+                self.service.create_grafana_team(team_name)
+            elif topic == "project" and key == "deleted":
+                log.debug("Received message from kafka for deleting team")
+                project_name = values["name"]
+                self.service.delete_grafana_team(project_name)
+            elif topic == "project" and key == "edited":
+                log.debug("Received message from kafka for team name update")
+                project_old_name = values["original"]["name"]
+                project_new_name = values["changes"]["name"]
+                self.service.update_grafana_team(project_new_name, project_old_name)
+        except Exception:
+            log.exception("Exception processing message: ")

 
     def dashboard_forever(self):
         log.debug('dashboard_forever')
 
     def dashboard_forever(self):
         log.debug('dashboard_forever')

diff --git a/osm_mon/dashboarder/service.py b/osm_mon/dashboarder/service.py
index 14ea0b9..2abd6be 100644 (file)
--- a/osm_mon/dashboarder/service.py
+++ b/osm_mon/dashboarder/service.py
@@ -69,13 +69,16 @@ class DashboarderService:
             constituent_vnfds = nsr['nsd']['constituent-vnfd']
             for constituent_vnfd in constituent_vnfds:
                 try:
             constituent_vnfds = nsr['nsd']['constituent-vnfd']
             for constituent_vnfd in constituent_vnfds:
                 try:

-                    vnfd = self.common_db.get_vnfd_by_id(constituent_vnfd['vnfd-id-ref'])
+                    vnfd = self.common_db.get_vnfd_by_name(constituent_vnfd['vnfd-id-ref'])

                     # If there are metrics, create dashboard (if exists)
                     # If there are metrics, create dashboard (if exists)

-                    if 'monitoring-param' in vnfd:
+                    if vnfd and 'monitoring-param' in vnfd:

                         if nsr_id not in dashboard_uids:
                             nsr_name = nsr['name']
                         if nsr_id not in dashboard_uids:
                             nsr_name = nsr['name']

+                            project_id = nsr["_admin"]["projects_read"][0]
+                            project_details = self.common_db.get_project(project_id)
+                            project_name = project_details["name"]

                             self.grafana.create_dashboard(nsr_id, nsr_name,
                             self.grafana.create_dashboard(nsr_id, nsr_name,

-                                                          dashboard_path)
+                                                          dashboard_path, project_name)

                             log.debug('Created dashboard for NS: %s', nsr_id)
                         else:
                             log.debug('Dashboard already exists')
                             log.debug('Created dashboard for NS: %s', nsr_id)
                         else:
                             log.debug('Dashboard already exists')


@@ -93,3 +96,29 @@ class DashboarderService:
                 log.debug('Deleted obsolete dashboard: %s', dashboard_uid)
             else:
                 log.debug('All dashboards in use')
                 log.debug('Deleted obsolete dashboard: %s', dashboard_uid)
             else:
                 log.debug('All dashboards in use')

+
+    def create_grafana_user(self, user):
+        self.grafana.create_grafana_users(user)
+
+    def create_grafana_team_member(self, project_data, userid):
+        user = self.common_db.get_user_by_id(userid)
+        user_name = user["username"]
+        proj_list = []
+        for project in project_data:
+            proj_list.append(project["project"])
+        for proj in project_data:
+            role_obj = self.common_db.get_role_by_name(proj["role"])
+            is_admin = role_obj["permissions"].get("admin")
+            self.grafana.create_grafana_teams_members(proj["project"], user_name, is_admin, proj_list)
+
+    def create_grafana_team(self, team_name):
+        self.grafana.create_grafana_teams(team_name)
+
+    def delete_grafana_user(self, user_name):
+        self.grafana.delete_grafana_users(user_name)
+
+    def delete_grafana_team(self, project_name):
+        self.grafana.delete_grafana_team(project_name)
+
+    def update_grafana_team(self, project_new_name, project_old_name):
+        self.grafana.update_grafana_teams(project_new_name, project_old_name)