Initial CloudWatch-boto plugin for MON 17/2117/1
authorWajeeha <wajeeha.hamid@xflowresearch.com>
Thu, 31 Aug 2017 11:52:29 +0000 (16:52 +0500)
committerWajeeha <wajeeha.hamid@xflowresearch.com>
Thu, 31 Aug 2017 11:52:29 +0000 (16:52 +0500)
Change-Id: I221db8635d768c41733b7ee0416b948b1862e04b
Signed-off-by: Wajeeha <wajeeha.hamid@xflowresearch.com>
plugins/CloudWatch/connection.py [new file with mode: 0644]
plugins/CloudWatch/metric_alarms.py [new file with mode: 0644]
plugins/CloudWatch/plugin.py [new file with mode: 0644]

diff --git a/plugins/CloudWatch/connection.py b/plugins/CloudWatch/connection.py
new file mode 100644 (file)
index 0000000..666879a
--- /dev/null
@@ -0,0 +1,90 @@
+##
+# Copyright 2017 xFlow Research Pvt. Ltd
+# This file is part of MON module
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact with: wajeeha.hamid@xflowresearch.com
+##
+
+'''
+AWS-Plugin implements all the methods of MON to interact with AWS using the BOTO client
+'''
+
+__author__ = "Wajeeha Hamid"
+__date__   = "31-August-2017"
+
+import sys
+import os
+
+try:
+    import boto
+    import boto.ec2
+    import boto.vpc
+    import boto.ec2.cloudwatch
+    import boto.ec2.connection
+    import logging as log
+    from boto.ec2.cloudwatch.alarm import MetricAlarm
+    from boto.ec2.cloudwatch.dimension import Dimension
+    from boto.sns import connect_to_region
+    from boto.utils import get_instance_metadata
+
+except:
+    exit("Boto not avialable. Try activating your virtualenv OR `pip install boto`")
+
+
+class Connection():
+    """Connection Establishement with AWS -- VPC/EC2/CloudWatch"""
+#-----------------------------------------------------------------------------------------------------------------------------
+    def setEnvironment(self):   
+
+        """Credentials for connecting to AWS-CloudWatch""" 
+        self.AWS_KEY = os.environ.get("AWS_ACCESS_KEY_ID")
+        self.AWS_SECRET = os.environ.get("AWS_SECRET_ACCESS_KEY")
+        self.AWS_REGION = os.environ.get("AWS_EC2_REGION","us-west-2")
+        #TOPIC = 'YOUR_TOPIC'
+#-----------------------------------------------------------------------------------------------------------------------------
+    def connection_instance(self):
+            try:
+                #VPC Connection
+                self.vpc_conn = boto.vpc.connect_to_region(self.AWS_REGION,
+                    aws_access_key_id=self.AWS_KEY,
+                    aws_secret_access_key=self.AWS_SECRET)
+                print self.vpc_conn
+                
+                #EC2 Connection
+                self.ec2_conn = boto.ec2.connect_to_region(self.AWS_REGION,
+                    aws_access_key_id=self.AWS_KEY,
+                    aws_secret_access_key=self.AWS_SECRET)
+                print self.ec2_conn
+                
+                """ TODO : Required to add actions against alarms when needed """
+                #self.sns = connect_to_region(self.AWS_REGION)
+                #self.topics = self.sns.get_all_topics()
+                #self.topic = self.topics[u'ListTopicsResponse']['ListTopicsResult']['Topics'][0]['TopicArn']
+
+                #Cloudwatch Connection
+                self.cloudwatch_conn = boto.ec2.cloudwatch.connect_to_region(
+                    self.AWS_REGION,
+                    aws_access_key_id=self.AWS_KEY,
+                    aws_secret_access_key=self.AWS_SECRET) 
+
+                return self.cloudwatch_conn
+                print "--- Connection Established with AWS ---"
+                print "\n"
+                
+            except Exception as e:
+                log.error("Failed to Connect with AWS %s: ",str(e)) 
+
diff --git a/plugins/CloudWatch/metric_alarms.py b/plugins/CloudWatch/metric_alarms.py
new file mode 100644 (file)
index 0000000..c58987d
--- /dev/null
@@ -0,0 +1,176 @@
+import sys
+import os
+import re
+import datetime
+import random
+import logging as log
+from random import randint
+from operator import itemgetter
+from connection import Connection
+
+try:
+    import boto
+    import boto.ec2
+    import boto.vpc
+    import boto.ec2.cloudwatch
+    import boto.ec2.connection
+except:
+    exit("Boto not avialable. Try activating your virtualenv OR `pip install boto`")
+
+
+class MetricAlarm():
+    """Alarms Functionality Handler -- Cloudwatch """
+    def config_alarm(self,cloudwatch_conn,alarm_info):
+       """Configure or Create a new alarm"""
+
+        """ Alarm Name to ID Mapping """
+        alarm_id = alarm_info['alarm_name'] + "_" + alarm_info['resource_id']
+        if self.is_present(cloudwatch_conn,alarm_id) == True: 
+            alarm_id = None
+            log.debug ("Alarm already exists, Try updating the alarm using 'update_alarm_configuration()'")   
+        else:              
+            try:
+                alarm = boto.ec2.cloudwatch.alarm.MetricAlarm(
+                    connection = cloudwatch_conn,
+                    name = alarm_info['alarm_name'] + "_" + alarm_info['resource_id'],
+                    metric = alarm_info['alarm_metric'],
+                    namespace = alarm_info['instance_type'],
+                    statistic = alarm_info['alarm_statistics'],
+                    comparison = alarm_info['alarm_comparison'],
+                    threshold = alarm_info['alarm_threshold'],
+                    period = alarm_info['alarm_period'],
+                    evaluation_periods = alarm_info['alarm_evaluation_period'],
+                    unit=alarm_info['alarm_unit'],
+                    description = alarm_info['alarm_severity'] + ";" + alarm_info['alarm_description'],
+                    dimensions = {'InstanceId':alarm_info['resource_id']},
+                    alarm_actions = None,
+                    ok_actions = None,
+                    insufficient_data_actions = None)
+
+                """Setting Alarm Actions : 
+                alarm_actions = ['arn:aws:swf:us-west-2:465479087178:action/actions/AWS_EC2.InstanceId.Stop/1.0']"""
+
+                cloudwatch_conn.put_metric_alarm(alarm)
+                log.debug ("Alarm Configured Succesfully")
+                print "created"
+                print "\n"    
+            except Exception as e:
+                log.error("Alarm Configuration Failed: " + str(e))
+        return alarm_id    
+#-----------------------------------------------------------------------------------------------------------------------------
+    def update_alarm(self,cloudwatch_conn,alarm_info):
+
+       """Update or reconfigure an alarm"""
+        
+        """Alarm Name to ID Mapping"""
+        alarm_id = alarm_info['alarm_name'] + "_" + alarm_info['resource_id']
+
+        """Verifying : Alarm exists already"""
+        if self.is_present(cloudwatch_conn,alarm_id) == False: 
+            alarm_id = None
+            log.debug("Alarm not found, Try creating the alarm using 'configure_alarm()'")   
+        else:            
+            try:
+                alarm = boto.ec2.cloudwatch.alarm.MetricAlarm(
+                        connection = cloudwatch_conn,
+                        name = alarm_info['alarm_name'] + "_" + alarm_info['resource_id'],
+                        metric = alarm_info['alarm_metric'],
+                        namespace = alarm_info['instance_type'],
+                        statistic = alarm_info['alarm_statistics'],
+                        comparison = alarm_info['alarm_comparison'],
+                        threshold = alarm_info['alarm_threshold'],
+                        period = alarm_info['alarm_period'],
+                        evaluation_periods = alarm_info['alarm_evaluation_period'],
+                        unit=alarm_info['alarm_unit'],
+                        description = alarm_info['alarm_severity'] + ";" + alarm_info['alarm_description'],
+                        dimensions = {'InstanceId':alarm_info['resource_id']},
+                        alarm_actions = None,
+                        ok_actions = None,
+                        insufficient_data_actions = None)
+                cloudwatch_conn.put_metric_alarm(alarm)
+                log.debug("Alarm %s Updated ",alarm.name)
+                print "updated"
+            except Exception as e:
+                log.error ("Error in Updating Alarm " + str(e))
+        return alarm_id
+#-----------------------------------------------------------------------------------------------------------------------------
+    def delete_Alarm(self,cloudwatch_conn,alarm_id):
+       """Deletes an Alarm with specified alarm_id"""
+        try:
+            if self.is_present(cloudwatch_conn,alarm_id) == True:
+                deleted_alarm=cloudwatch_conn.delete_alarms(alarm_id)
+                return alarm_id
+            return None 
+        except Exception as e:
+                log.error("Alarm Not Deleted: " + str(e))      
+#-----------------------------------------------------------------------------------------------------------------------------
+    def alarms_list(self,cloudwatch_conn,instance_id):
+
+       """Get a list of alarms that are present on a particular VM instance"""
+        try:
+            log.debug("Getting Alarm list for %s",instance_id)
+            alarm_dict = dict()
+            alarm_list = []
+            alarms = cloudwatch_conn.describe_alarms()
+            itr = 0
+            for alarm in alarms:
+                if str(alarm.dimensions['InstanceId']).split("'")[1] == instance_id:
+                    alarm_list.insert(itr,str(alarm.name))
+                    itr += 1
+            alarm_dict['alarm_names'] = alarm_list
+            alarm_dict['resource_id'] = instance_id   
+            return alarm_dict
+        except Exception as e:
+                log.error("Error in Getting List : %s",str(e))    
+#-----------------------------------------------------------------------------------------------------------------------------
+    def alarm_details(self,cloudwatch_conn,alarm_name):
+
+       """Get an individual alarm details specified by alarm_name"""
+        try:
+            alarms_details=cloudwatch_conn.describe_alarm_history()       
+            alarm_details_dict = dict()
+            
+            for itr in range (len(alarms_details)):
+                if alarms_details[itr].name == alarm_name and 'created' in alarms_details[itr].summary :#name, timestamp, summary
+                    status = alarms_details[itr].summary.split()                   
+                    alarms = cloudwatch_conn.describe_alarms()
+                    for alarm in alarms:
+                        if alarm.name == alarm_name:
+                            alarm_details_dict['alarm_id'] = alarm_name
+                            alarm_details_dict['resource_id'] = str(alarm.dimensions['InstanceId']).split("'")[1]
+                            alarm_details_dict['severity'] = str(alarm.description)
+                            alarm_details_dict['start_date_time'] = str(alarms_details[x].timestamp) 
+
+                            return alarm_details_dict             
+                  
+        except Exception as e:
+               log.error("Error getting alarm details: %s",str(e))           
+#-----------------------------------------------------------------------------------------------------------------------------
+    def metrics_data(self,cloudwatch_conn,metric_name,instance_id,period,metric_unit):
+
+       """Getting Metrics Stats for an Hour. Time interval can be modified using Timedelta value"""
+        metric_data= dict()
+        metric_stats=cloudwatch_conn.get_metric_statistics(period, datetime.datetime.utcnow() - datetime.timedelta(seconds=3600),
+                            datetime.datetime.utcnow(),metric_name,'AWS/EC2', 'Maximum',
+                            dimensions={'InstanceId':instance_id}, unit=metric_unit)
+
+        for itr in range (len(metric_stats)):
+            metric_data['metric_name'] = metric_name
+            metric_data['Resource_id'] = instance_id
+            metric_data['Unit']                   = metric_stats[itr]['Unit']
+            metric_data['Timestamp']   = metric_stats[itr]['Timestamp']  
+        return metric_data
+
+#-----------------------------------------------------------------------------------------------------------------------------
+    def is_present(self,cloudwatch_conn,alarm_name):
+       """Finding Alarm exists or not"""
+        try:
+            alarms = cloudwatch_conn.describe_alarms()
+            for alarm in alarms:
+                if alarm.name == alarm_name:
+                    return True
+            return False
+        except Exception as e:
+                log.error("Error Finding Alarm",str(e))             
+#-----------------------------------------------------------------------------------------------------------------------------
diff --git a/plugins/CloudWatch/plugin.py b/plugins/CloudWatch/plugin.py
new file mode 100644 (file)
index 0000000..9f917ae
--- /dev/null
@@ -0,0 +1,134 @@
+##
+# Copyright 2017 xFlow Research Pvt. Ltd
+# This file is part of MON module
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# For those usages not covered by the Apache License, Version 2.0 please
+# contact with: wajeeha.hamid@xflowresearch.com
+##
+
+'''
+AWS-Plugin implements all the methods of MON to interact with AWS using the BOTO client
+'''
+
+__author__ = "Wajeeha Hamid"
+__date__   = "31-August-2017"
+
+import sys
+from connection import Connection
+from metric_alarms import MetricAlarm
+try:
+    from kafka import KafkaConsumer
+    from kafka.errors import KafkaError
+except:
+    exit("Kafka Error. Try activating your Kafka Services")
+
+class Plugin():
+    """Receives Alarm info from MetricAlarm and connects with the consumer/producer """
+    def __init__ (self): 
+        self.conn = Connection()
+        self.metricAlarm = MetricAlarm()
+
+        server = {'server': 'localhost:9092', 'topic': 'alarms'}
+    #Initialize a Consumer object to consume message from the SO    
+        self._consumer = KafkaConsumer(server['topic'],
+                                       group_id='my-group',
+                                       bootstrap_servers=server['server'])
+#---------------------------------------------------------------------------------------------------------------------------      
+    def connection(self):
+        """Connecting instances with CloudWatch"""
+        self.conn.setEnvironment()
+        self.cloudwatch_conn = self.conn.connection_instance()
+#---------------------------------------------------------------------------------------------------------------------------   
+    def configure_alarm(self,alarm_info):
+
+        alarm_id = self.metricAlarm.config_alarm(self.cloudwatch_conn,alarm_info)
+        return alarm_id
+#---------------------------------------------------------------------------------------------------------------------------          
+    def update_alarm_configuration(self,test):
+        alarm_id = self.metricAlarm.update_alarm(self.cloudwatch_conn,test)
+        return alarm_id
+#---------------------------------------------------------------------------------------------------------------------------            
+    def delete_alarm(self,alarm_id):
+        return self.metricAlarm.delete_Alarm(self.cloudwatch_conn,alarm_id)
+#---------------------------------------------------------------------------------------------------------------------------  
+    def get_alarms_list(self,instance_id):
+        return self.metricAlarm.alarms_list(self.cloudwatch_conn,instance_id) 
+#---------------------------------------------------------------------------------------------------------------------------            
+    def get_alarm_details(self,alarm_id):
+        return self.metricAlarm.alarm_details(self.cloudwatch_conn,alarm_id)
+#---------------------------------------------------------------------------------------------------------------------------            
+    def get_metrics_data(self,metric_name,instance_id,period,metric_unit):
+        return self.metricAlarm.metrics_data(self.cloudwatch_conn,metric_name,instance_id,period,metric_unit)
+#---------------------------------------------------------------------------------------------------------------------------   
+    def consumer(self,alarm_info):
+        try:
+            for message in self._consumer:
+
+                # Check the Functionlity that needs to be performed: topic = 'alarms'/'metrics'/'Access_Credentials'
+                if message.topic == "alarms":
+                    log.info("Action required against: %s" % (message.topic))
+
+                    if message.key == "Configure_Alarm":
+                        #alarm_info = json.loads(message.value)
+                        alarm_id = self.configure_alarm(alarm_info) #alarm_info = message.value
+                        log.info("New alarm created with alarmID: %s", alarm_id)
+                #Keys other than Configure_Alarm and Notify_Alarm are already handled here which are not yet finalized
+                    elif message.key == "Notify_Alarm":
+                        alarm_details = self.get_alarm_details(alarm_info['alarm_name'])#['alarm_id']
+
+                    elif message.key == "Update_Alarm":
+                        alarm_id = self.update_alarm_configuration(alarm_info)
+                        log.info("Alarm Updated with alarmID: %s", alarm_id)
+                    
+                    elif message.key == "Delete_Alarm":    
+                        alarm_id = self.delete_alarm(alarm_info['alarm_name'])
+                        log.info("Alarm Deleted with alarmID: %s", alarm_id)
+                   
+                    elif message.key == "Alarms_List":    
+                        self.get_alarms_list(alarm_info['resource_id'])#['alarm_names']
+                    else:
+                        log.debug("Unknown key, no action will be performed")
+                else:
+                    log.info("Message topic not relevant to this plugin: %s",
+                         message.topic)    
+        except Exception as e:
+                log.error("Consumer exception: %s", str(e))             
+#---------------------------------------------------------------------------------------------------------------------------   
+"""For Testing Purpose: Required calls to Trigger defined functions """
+'''obj = Plugin()
+obj.connection() 
+obj.consumer()
+
+alarm_info = dict()
+alarm_info['resource_id'] = 'i-098da78cbd8304e17'
+alarm_info['alarm_name'] = 'alarm-6'
+alarm_info['alarm_metric'] = 'CPUUtilization'
+alarm_info['alarm_severity'] = 'Critical'
+alarm_info['instance_type'] = 'AWS/EC2'
+alarm_info['alarm_statistics'] = 'Maximum'
+alarm_info['alarm_comparison'] = '>='
+alarm_info['alarm_threshold'] = 1.5
+alarm_info['alarm_period'] = 60
+alarm_info['alarm_evaluation_period'] = 1
+alarm_info['alarm_unit'] = None
+alarm_info['alarm_description'] = ''
+
+#obj.configure_alarm(alarm_info)
+#obj.update_alarm_configuration(alarm_info)
+#obj.delete_alarm('alarm-5|i-098da78cbd8304e17')
+#obj.get_alarms_list('i-098da78cbd8304e17')#['alarm_names']
+#obj.get_alarm_details('alarm-5|i-098da78cbd8304e17')#['alarm_id']
+#print obj.get_metrics_data('CPUUtilization','i-09462760703837b26','60',None)      '''
\ No newline at end of file