From: stevenvanrossem Date: Fri, 13 May 2016 13:09:20 +0000 (+0200) Subject: Merge remote-tracking branch 'upstream/master' X-Git-Tag: v3.1~124^2~2 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fvim-emu.git;a=commitdiff_plain;h=5e040bf29bd5cda68f1416b48227a7419cb6fee0;hp=2b7765f2750a8bedeb3d7cab62a7c6df2fd15d78 Merge remote-tracking branch 'upstream/master' --- diff --git a/misc/prometheus_jenkins.yml b/misc/prometheus_jenkins.yml new file mode 100755 index 0000000..b7770d1 --- /dev/null +++ b/misc/prometheus_jenkins.yml @@ -0,0 +1,26 @@ +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + evaluation_interval: 5s + + +# A scrape configuration containing endpoints to scrape: + +scrape_configs: + +# cAdvsior started in son-emu + - job_name: 'cAdvisor' + + scrape_interval: 1s + + target_groups: + - targets: ['172.17.0.1:8090'] + +# Pushgateway started in SP + - job_name: 'PushGateway' + + scrape_interval: 1s + + target_groups: + - targets: ['172.17.0.1:9091'] + + diff --git a/setup_cli_only.py b/setup_cli_only.py new file mode 100755 index 0000000..119cf8c --- /dev/null +++ b/setup_cli_only.py @@ -0,0 +1,25 @@ +from setuptools import setup, find_packages + +setup(name='emuvim', + version='0.0.1', + license='Apache 2.0', + description='emuvim is a VIM for the SONATA platform', + url='http://github.com/sonata-emu', + author_email='sonata-dev@sonata-nfv.eu', + package_dir={'': 'src'}, + # packages=find_packages('emuvim', exclude=['*.test', '*.test.*', 'test.*', 'test']), + packages=find_packages('src'), + install_requires=[ + 'zerorpc', + 'tabulate', + 'argparse', + ], + zip_safe=False, + entry_points={ + 'console_scripts': [ + 'son-emu-cli=emuvim.cli.son_emu_cli:main', + ], + }, + setup_requires=['pytest-runner'], + tests_require=['pytest'], +) \ No newline at end of file diff --git a/src/emuvim/api/zerorpc/network.py b/src/emuvim/api/zerorpc/network.py index 37b34f9..d55e775 100755 --- a/src/emuvim/api/zerorpc/network.py +++ b/src/emuvim/api/zerorpc/network.py @@ -134,6 +134,16 @@ class DCNetworkApi(object): logging.exception("RPC error.") return ex.message + # remove the flow metrics measurement + def stop_flow(self, vnf_name, vnf_interface, metric, cookie): + logging.debug("RPC CALL: stop flow") + try: + c = self.net.monitor_agent.stop_flow(vnf_name, vnf_interface, metric, cookie) + return c + except Exception as ex: + logging.exception("RPC error.") + return ex.message + # do prometheus query def prometheus(self, dc_label, vnf_name, vnf_interface, query): logging.debug("RPC CALL: query prometheus") diff --git a/src/emuvim/cli/compute.py b/src/emuvim/cli/compute.py index dcb499d..d01dfed 100755 --- a/src/emuvim/cli/compute.py +++ b/src/emuvim/cli/compute.py @@ -30,7 +30,6 @@ class ZeroRpcClient(object): nw_list = list() if args.get("network") is not None: nw_list = self._parse_network(args.get("network")) - r = self.c.compute_action_start( args.get("datacenter"), args.get("name"), diff --git a/src/emuvim/cli/monitor.py b/src/emuvim/cli/monitor.py index 123abe5..79ff25b 100755 --- a/src/emuvim/cli/monitor.py +++ b/src/emuvim/cli/monitor.py @@ -5,20 +5,24 @@ son-emu monitor CLI import argparse import pprint -from tabulate import tabulate import zerorpc -import time - +import prometheus pp = pprint.PrettyPrinter(indent=4) class ZeroRpcClient(object): def __init__(self): + # network zerorpc self.c = zerorpc.Client() # TODO connect to DCNetwork API #self.c.connect("tcp://127.0.0.1:4242") # TODO hard coded for now. we'll change this later self.c.connect("tcp://127.0.0.1:5151") + + # compute zerorpc + self.compute_api = zerorpc.Client(heartbeat=None, timeout=120) # heartbeat=None, timeout=120 + self.compute_api.connect("tcp://127.0.0.1:4242") # TODO hard coded for now. we'll change this later + self.cmds = {} def execute_command(self, args): @@ -56,7 +60,17 @@ class ZeroRpcClient(object): args.get("cookie")) pp.pprint(r) - def prometheus(self, args): + def stop_flow(self, args): + vnf_name = self._parse_vnf_name(args.get("vnf_name")) + vnf_interface = self._parse_vnf_interface(args.get("vnf_name")) + r = self.c.stop_flow( + vnf_name, + vnf_interface, + args.get("metric"), + args.get("cookie")) + pp.pprint(r) + + def prometheus_zrpc(self, args): vnf_name = self._parse_vnf_name(args.get("vnf_name")) vnf_interface = self._parse_vnf_interface(args.get("vnf_name")) r = self.c.prometheus( @@ -66,6 +80,18 @@ class ZeroRpcClient(object): args.get("query")) pp.pprint(r) + def prometheus(self, args): + vnf_name = self._parse_vnf_name(args.get("vnf_name")) + vnf_interface = self._parse_vnf_interface(args.get("vnf_name")) + dc_label = args.get("datacenter") + query = args.get("query") + vnf_status = self.compute_api.compute_status(dc_label, vnf_name) + uuid = vnf_status['id'] + query = query.replace('', uuid) + + r = prometheus.query_Prometheus(query) + pp.pprint(r) + def _parse_vnf_name(self, vnf_name_str): vnf_name = vnf_name_str.split(':')[0] @@ -82,7 +108,8 @@ class ZeroRpcClient(object): parser = argparse.ArgumentParser(description='son-emu monitor') parser.add_argument( "command", - help="Action to be executed") + choices=['setup_metric', 'stop_metric', 'setup_flow', 'stop_flow','prometheus'], + help="setup/stop a metric/flow to be monitored or query Prometheus") parser.add_argument( "--vnf_name", "-vnf", dest="vnf_name", help="vnf name:interface to be monitored") diff --git a/src/emuvim/cli/network.py b/src/emuvim/cli/network.py index 53007cd..516a752 100755 --- a/src/emuvim/cli/network.py +++ b/src/emuvim/cli/network.py @@ -84,7 +84,8 @@ class ZeroRpcClient(object): parser = argparse.ArgumentParser(description='son-emu network') parser.add_argument( "command", - help="Action to be executed: add|remove") + choices=['add', 'remove'], + help="Action to be executed.") parser.add_argument( "--datacenter", "-d", dest="datacenter", help="Data center to in which the network action should be initiated") @@ -103,10 +104,10 @@ parser.add_argument( parser.add_argument( "--bidirectional", "-b", dest="bidirectional", action='store_true', - help="add/remove the flow entries in 2 directions") + help="add/remove the flow entries from src to dst and back") parser.add_argument( "--cookie", "-c", dest="cookie", - help="cookie for this flow") + help="cookie for this flow, as easy to use identifier (eg. per tenant/service)") def main(argv): args = vars(parser.parse_args(argv)) diff --git a/src/emuvim/cli/prometheus.py b/src/emuvim/cli/prometheus.py new file mode 100755 index 0000000..8d87d89 --- /dev/null +++ b/src/emuvim/cli/prometheus.py @@ -0,0 +1,28 @@ +""" +Prometheus API helper functions +(c) 2016 by Steven Van Rossem +""" + +import urllib2 +import ast + +prometheus_ip = '0.0.0.0' +prometheus_port = '9090' +prometheus_REST_api = 'http://{0}:{1}'.format(prometheus_ip, prometheus_port) + + +def query_Prometheus(query): + url = prometheus_REST_api + '/' + 'api/v1/query?query=' + query + # logging.info('query:{0}'.format(url)) + req = urllib2.Request(url) + ret = urllib2.urlopen(req).read() + ret = ast.literal_eval(ret) + if ret['status'] == 'success': + # logging.info('return:{0}'.format(ret)) + try: + ret = ret['data']['result'][0]['value'] + except: + ret = None + else: + ret = None + return ret \ No newline at end of file diff --git a/src/emuvim/dcemulator/monitoring.py b/src/emuvim/dcemulator/monitoring.py index 762c947..8db10da 100755 --- a/src/emuvim/dcemulator/monitoring.py +++ b/src/emuvim/dcemulator/monitoring.py @@ -75,8 +75,8 @@ class DCNetworkMonitor(): self.monitor_flow_thread.start() # helper tools - self.pushgateway_process = self.start_PushGateway() - self.prometheus_process = self.start_Prometheus() + #self.pushgateway_process = self.start_PushGateway() + #self.prometheus_process = self.start_Prometheus() self.cadvisor_process = self.start_cadvisor() # first set some parameters, before measurement can start @@ -139,6 +139,28 @@ class DCNetworkMonitor(): logging.exception("setup_metric error.") return ex.message + def stop_flow(self, vnf_name, vnf_interface=None, metric=None, cookie=0): + for flow_dict in self.flow_metrics: + if flow_dict['vnf_name'] == vnf_name and flow_dict['vnf_interface'] == vnf_interface \ + and flow_dict['metric_key'] == metric and flow_dict['cookie'] == cookie: + + self.monitor_flow_lock.acquire() + + self.flow_metrics.remove(flow_dict) + + for collector in self.registry._collectors: + if (vnf_name, vnf_interface, cookie) in collector._metrics: + #logging.info('2 name:{0} labels:{1} metrics:{2}'.format(collector._name, collector._labelnames, + # collector._metrics)) + collector.remove(vnf_name, vnf_interface, cookie) + + delete_from_gateway(self.pushgateway, job='sonemu-SDNcontroller') + + self.monitor_flow_lock.release() + + logging.info('Stopped monitoring flow {3}: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie)) + return 'Stopped monitoring flow {3}: {2} on {0}:{1}'.format(vnf_name, vnf_interface, metric, cookie) + # first set some parameters, before measurement can start def setup_metric(self, vnf_name, vnf_interface=None, metric='tx_packets'): @@ -297,7 +319,7 @@ class DCNetworkMonitor(): ret = self.net.ryu_REST('stats/flow', dpid=flow_dict['switch_dpid'], data=data) flow_stat_dict = ast.literal_eval(ret) - logging.info('received flow stat:{0} '.format(flow_stat_dict)) + #logging.info('received flow stat:{0} '.format(flow_stat_dict)) self.set_flow_metric(flow_dict, flow_stat_dict) self.monitor_flow_lock.release() @@ -474,6 +496,7 @@ class DCNetworkMonitor(): self.monitor_thread.join() self.monitor_flow_thread.join() + ''' if self.prometheus_process is not None: logging.info('stopping prometheus container') self.prometheus_process.terminate() @@ -485,6 +508,7 @@ class DCNetworkMonitor(): self.pushgateway_process.terminate() self.pushgateway_process.kill() self._stop_container('pushgateway') + ''' if self.cadvisor_process is not None: logging.info('stopping cadvisor container') diff --git a/src/emuvim/dcemulator/net.py b/src/emuvim/dcemulator/net.py index 3556535..39c4a96 100755 --- a/src/emuvim/dcemulator/net.py +++ b/src/emuvim/dcemulator/net.py @@ -236,15 +236,17 @@ class DCNetwork(Containernet): if cmd == 'add-flow': ret = self._chainAddFlow(vnf_src_name, vnf_dst_name, vnf_src_interface, vnf_dst_interface, **kwargs) if kwargs.get('bidirectional'): - return ret +'\n' + self._chainAddFlow(vnf_dst_name, vnf_src_name, vnf_dst_interface, vnf_src_interface, **kwargs) + ret = ret +'\n' + self._chainAddFlow(vnf_dst_name, vnf_src_name, vnf_dst_interface, vnf_src_interface, **kwargs) elif cmd == 'del-flows': # TODO: del-flow to be implemented ret = self._chainAddFlow(vnf_src_name, vnf_dst_name, vnf_src_interface, vnf_dst_interface, **kwargs) if kwargs.get('bidirectional'): - return ret + '\n' + self._chainAddFlow(vnf_dst_name, vnf_src_name, vnf_dst_interface, vnf_src_interface, **kwargs) + ret = ret + '\n' + self._chainAddFlow(vnf_dst_name, vnf_src_name, vnf_dst_interface, vnf_src_interface, **kwargs) else: - return "Command unknown" + ret = "Command unknown" + + return ret def _chainAddFlow(self, vnf_src_name, vnf_dst_name, vnf_src_interface=None, vnf_dst_interface=None, **kwargs): @@ -404,9 +406,8 @@ class DCNetwork(Containernet): elif cmd == 'del-flows': prefix = 'stats/flowentry/delete' - # if cookie is given, only delete flows by cookie - # do not specify other match -> also other cookies can be matched if cookie: + # TODO: add cookie_mask as argument flow['cookie_mask'] = int('0xffffffffffffffff', 16) # need full mask to match complete cookie action = {} @@ -479,7 +480,6 @@ class DCNetwork(Containernet): self.ryu_process.kill() def ryu_REST(self, prefix, dpid=None, data=None): - if data: logging.info('log POST: {0}'.format(str(data))) try: if dpid: url = self.ryu_REST_api + '/' + str(prefix) + '/' + str(dpid) diff --git a/src/emuvim/examples/monitoring_demo_topology.py b/src/emuvim/examples/monitoring_demo_topology.py index 4dfd5b7..0650be4 100755 --- a/src/emuvim/examples/monitoring_demo_topology.py +++ b/src/emuvim/examples/monitoring_demo_topology.py @@ -32,11 +32,16 @@ def create_topology1(): net = DCNetwork(monitor=True, enable_learning=False) """ - 1b. add a monitoring agent to the DCNetwork + 1b. Add endpoint APIs for the whole DCNetwork, + to access and control the networking from outside. + e.g., to setup forwarding paths between compute + instances aka. VNFs (represented by Docker containers), passing through + different switches and datacenters of the emulated topology """ mon_api = ZeroRpcApiEndpointDCNetwork("0.0.0.0", 5151) mon_api.connectDCNetwork(net) mon_api.start() + """ 2. Add (logical) data centers to the topology (each data center is one "bigswitch" in our simplified diff --git a/src/emuvim/test/integrationtests/test_sdk_monitor.sh b/src/emuvim/test/integrationtests/test_sdk_monitor.sh new file mode 100755 index 0000000..26f939f --- /dev/null +++ b/src/emuvim/test/integrationtests/test_sdk_monitor.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# test if a vnf can be monitored and deployed + +cpu_load=$(son-emu-cli monitor prometheus -d datacenter1 -vnf vnf1 -q 'sum(rate(container_cpu_usage_seconds_total{id="/docker/"}[10s]))') + +# test if prometheus query worked +regex="[0-9.]+, [0-9.']+" +if [[ $cpu_load =~ $regex ]] ; then + echo "OK" + exit 0 +else + echo $cpu_load + echo "not OK" + exit 1 +fi