21985cba695edd456d9847c16626ae49517bb220
2 Copyright (c) 2015 SONATA-NFV
5 Licensed under the Apache License, Version 2.0 (the "License");
6 you may not use this file except in compliance with the License.
7 You may obtain a copy of the License at
9 http://www.apache.org/licenses/LICENSE-2.0
11 Unless required by applicable law or agreed to in writing, software
12 distributed under the License is distributed on an "AS IS" BASIS,
13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 See the License for the specific language governing permissions and
15 limitations under the License.
17 Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
18 nor the names of its contributors may be used to endorse or promote
19 products derived from this software without specific prior written
22 This work has been performed in the framework of the SONATA project,
23 funded by the European Commission under Grant number 671517 through
24 the Horizon 2020 and 5G-PPP programmes. The authors would like to
25 acknowledge the contributions of their colleagues of the SONATA
26 partner consortium (www.sonata-nfv.eu).
31 from mininet
.node
import OVSSwitch
34 from prometheus_client
import start_http_server
, Summary
, Histogram
, Gauge
, Counter
, REGISTRY
, CollectorRegistry
, \
35 pushadd_to_gateway
, push_to_gateway
, delete_from_gateway
37 from subprocess
import Popen
41 from copy
import deepcopy
46 class to read openflow stats from the Ryu controller of the DCNetwork
49 PUSHGATEWAY_PORT
= 9091
50 # we cannot use port 8080 because ryu-ofrest api is already using that one
53 COOKIE_MASK
= 0xffffffff
55 class DCNetworkMonitor():
56 def __init__(self
, net
):
58 self
.dockercli
= docker
.from_env()
61 self
.pushgateway
= 'localhost:{0}'.format(PUSHGATEWAY_PORT
)
63 # supported Prometheus metrics
64 self
.registry
= CollectorRegistry()
65 self
.prom_tx_packet_count
= Gauge('sonemu_tx_count_packets', 'Total number of packets sent',
66 ['vnf_name', 'vnf_interface', 'flow_id'], registry
=self
.registry
)
67 self
.prom_rx_packet_count
= Gauge('sonemu_rx_count_packets', 'Total number of packets received',
68 ['vnf_name', 'vnf_interface', 'flow_id'], registry
=self
.registry
)
69 self
.prom_tx_byte_count
= Gauge('sonemu_tx_count_bytes', 'Total number of bytes sent',
70 ['vnf_name', 'vnf_interface', 'flow_id'], registry
=self
.registry
)
71 self
.prom_rx_byte_count
= Gauge('sonemu_rx_count_bytes', 'Total number of bytes received',
72 ['vnf_name', 'vnf_interface', 'flow_id'], registry
=self
.registry
)
74 self
.prom_metrics
={'tx_packets':self
.prom_tx_packet_count
, 'rx_packets':self
.prom_rx_packet_count
,
75 'tx_bytes':self
.prom_tx_byte_count
,'rx_bytes':self
.prom_rx_byte_count
}
77 # list of installed metrics to monitor
78 # each entry can contain this data
84 previous_measurement = 0
85 previous_monitor_time = 0
90 self
.monitor_lock
= threading
.Lock()
91 self
.monitor_flow_lock
= threading
.Lock()
92 self
.network_metrics
= []
93 self
.flow_metrics
= []
94 self
.skewmon_metrics
= {}
96 # start monitoring thread
97 self
.start_monitoring
= True
98 self
.monitor_thread
= threading
.Thread(target
=self
.get_network_metrics
)
99 self
.monitor_thread
.start()
101 self
.monitor_flow_thread
= threading
.Thread(target
=self
.get_flow_metrics
)
102 self
.monitor_flow_thread
.start()
105 # cAdvisor, Prometheus pushgateway are started as external container, to gather monitoring metric in son-emu
106 self
.pushgateway_process
= self
.start_PushGateway()
107 self
.cadvisor_process
= self
.start_cAdvisor()
110 # first set some parameters, before measurement can start
111 def setup_flow(self
, vnf_name
, vnf_interface
=None, metric
='tx_packets', cookie
=0):
115 # check if port is specified (vnf:port)
116 if vnf_interface
is None:
117 # take first interface by default
118 connected_sw
= self
.net
.DCNetwork_graph
.neighbors(vnf_name
)[0]
119 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
120 vnf_interface
= link_dict
[0]['src_port_id']
122 flow_metric
['vnf_name'] = vnf_name
123 flow_metric
['vnf_interface'] = vnf_interface
126 for connected_sw
in self
.net
.DCNetwork_graph
.neighbors(vnf_name
):
127 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
128 for link
in link_dict
:
129 if link_dict
[link
]['src_port_id'] == vnf_interface
:
130 # found the right link and connected switch
131 vnf_switch
= connected_sw
132 flow_metric
['mon_port'] = link_dict
[link
]['dst_port_nr']
136 logging
.exception("vnf switch of {0}:{1} not found!".format(vnf_name
, vnf_interface
))
137 return "vnf switch of {0}:{1} not found!".format(vnf_name
, vnf_interface
)
140 # default port direction to monitor
142 metric
= 'tx_packets'
144 next_node
= self
.net
.getNodeByName(vnf_switch
)
146 if not isinstance(next_node
, OVSSwitch
):
147 logging
.info("vnf: {0} is not connected to switch".format(vnf_name
))
150 flow_metric
['previous_measurement'] = 0
151 flow_metric
['previous_monitor_time'] = 0
153 flow_metric
['switch_dpid'] = int(str(next_node
.dpid
), 16)
154 flow_metric
['metric_key'] = metric
155 flow_metric
['cookie'] = cookie
157 self
.monitor_flow_lock
.acquire()
158 self
.flow_metrics
.append(flow_metric
)
159 self
.monitor_flow_lock
.release()
161 logging
.info('Started monitoring flow:{3} {2} on {0}:{1}'.format(vnf_name
, vnf_interface
, metric
, cookie
))
162 return 'Started monitoring flow:{3} {2} on {0}:{1}'.format(vnf_name
, vnf_interface
, metric
, cookie
)
164 except Exception as ex
:
165 logging
.exception("setup_metric error.")
168 def stop_flow(self
, vnf_name
, vnf_interface
=None, metric
=None, cookie
=0,):
170 # check if port is specified (vnf:port)
171 if vnf_interface
is None and metric
is not None:
172 # take first interface by default
173 connected_sw
= self
.net
.DCNetwork_graph
.neighbors(vnf_name
)[0]
174 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
175 vnf_interface
= link_dict
[0]['src_port_id']
177 for flow_dict
in self
.flow_metrics
:
178 if flow_dict
['vnf_name'] == vnf_name
and flow_dict
['vnf_interface'] == vnf_interface \
179 and flow_dict
['metric_key'] == metric
and flow_dict
['cookie'] == cookie
:
181 self
.monitor_flow_lock
.acquire()
183 self
.flow_metrics
.remove(flow_dict
)
186 self
.prom_metrics
[flow_dict
['metric_key']]. \
187 labels(vnf_name
=vnf_name
, vnf_interface
=vnf_interface
, flow_id
=cookie
). \
190 delete_from_gateway(self
.pushgateway
, job
='sonemu-SDNcontroller')
192 self
.monitor_flow_lock
.release()
194 logging
.info('Stopped monitoring flow {3}: {2} on {0}:{1}'.format(vnf_name
, vnf_interface
, metric
, cookie
))
195 return 'Stopped monitoring flow {3}: {2} on {0}:{1}'.format(vnf_name
, vnf_interface
, metric
, cookie
)
197 return 'Error stopping monitoring flow: {0} on {1}:{2}'.format(metric
, vnf_name
, vnf_interface
)
200 # first set some parameters, before measurement can start
201 def setup_metric(self
, vnf_name
, vnf_interface
=None, metric
='tx_packets'):
205 # check if port is specified (vnf:port)
206 if vnf_interface
is None or vnf_interface
== '':
207 # take first interface by default
208 connected_sw
= self
.net
.DCNetwork_graph
.neighbors(vnf_name
)[0]
209 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
210 vnf_interface
= link_dict
[0]['src_port_id']
212 network_metric
['vnf_name'] = vnf_name
213 network_metric
['vnf_interface'] = vnf_interface
215 for connected_sw
in self
.net
.DCNetwork_graph
.neighbors(vnf_name
):
216 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
217 for link
in link_dict
:
218 if link_dict
[link
]['src_port_id'] == vnf_interface
:
219 # found the right link and connected switch
220 network_metric
['mon_port'] = link_dict
[link
]['dst_port_nr']
223 if 'mon_port' not in network_metric
:
224 logging
.exception("vnf interface {0}:{1} not found!".format(vnf_name
,vnf_interface
))
225 return "vnf interface {0}:{1} not found!".format(vnf_name
,vnf_interface
)
228 # default port direction to monitor
230 metric
= 'tx_packets'
232 vnf_switch
= self
.net
.DCNetwork_graph
.neighbors(str(vnf_name
))
234 if len(vnf_switch
) > 1:
235 logging
.info("vnf: {0} has multiple ports".format(vnf_name
))
237 elif len(vnf_switch
) == 0:
238 logging
.info("vnf: {0} is not connected".format(vnf_name
))
241 vnf_switch
= vnf_switch
[0]
242 next_node
= self
.net
.getNodeByName(vnf_switch
)
244 if not isinstance(next_node
, OVSSwitch
):
245 logging
.info("vnf: {0} is not connected to switch".format(vnf_name
))
248 network_metric
['previous_measurement'] = 0
249 network_metric
['previous_monitor_time'] = 0
252 network_metric
['switch_dpid'] = int(str(next_node
.dpid
), 16)
253 network_metric
['metric_key'] = metric
255 self
.monitor_lock
.acquire()
256 self
.network_metrics
.append(network_metric
)
257 self
.monitor_lock
.release()
260 logging
.info('Started monitoring: {2} on {0}:{1}'.format(vnf_name
, vnf_interface
, metric
))
261 return 'Started monitoring: {2} on {0}:{1}'.format(vnf_name
, vnf_interface
, metric
)
263 except Exception as ex
:
264 logging
.exception("setup_metric error.")
267 def stop_metric(self
, vnf_name
, vnf_interface
=None, metric
=None):
269 # check if port is specified (vnf:port)
270 if vnf_interface
is None and metric
is not None:
271 # take first interface by default
272 connected_sw
= self
.net
.DCNetwork_graph
.neighbors(vnf_name
)[0]
273 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
274 vnf_interface
= link_dict
[0]['src_port_id']
276 for metric_dict
in deepcopy(self
.network_metrics
):
277 if metric_dict
['vnf_name'] == vnf_name
and metric_dict
['vnf_interface'] == vnf_interface \
278 and metric_dict
['metric_key'] == metric
:
280 self
.monitor_lock
.acquire()
282 self
.network_metrics
.remove(metric_dict
)
284 # set values to NaN, prometheus api currently does not support removal of metrics
285 #self.prom_metrics[metric_dict['metric_key']].labels(vnf_name, vnf_interface).set(float('nan'))
286 self
.prom_metrics
[metric_dict
['metric_key']]. \
287 labels(vnf_name
=vnf_name
, vnf_interface
=vnf_interface
, flow_id
=None). \
290 # this removes the complete metric, all labels...
291 # 1 single monitor job for all metrics of the SDN controller
292 # we can only remove from the pushgateway grouping keys(labels) which we have defined for the add_to_pushgateway
293 # we can not specify labels from the metrics to be removed
294 # if we need to remove the metrics seperatelty, we need to give them a separate grouping key, and probably a diffferent registry also
295 delete_from_gateway(self
.pushgateway
, job
='sonemu-SDNcontroller')
297 self
.monitor_lock
.release()
299 logging
.info('Stopped monitoring: {2} on {0}:{1}'.format(vnf_name
, vnf_interface
, metric
))
300 return 'Stopped monitoring: {2} on {0}:{1}'.format(vnf_name
, vnf_interface
, metric
)
302 # delete everything from this vnf
303 elif metric_dict
['vnf_name'] == vnf_name
and vnf_interface
is None and metric
is None:
304 self
.monitor_lock
.acquire()
305 self
.network_metrics
.remove(metric_dict
)
306 logging
.info('remove metric from monitor: vnf_name:{0} vnf_interface:{1} mon_port:{2}'.format(metric_dict
['vnf_name'], metric_dict
['vnf_interface'], metric_dict
['mon_port']))
308 delete_from_gateway(self
.pushgateway
, job
='sonemu-SDNcontroller')
309 self
.monitor_lock
.release()
312 if vnf_interface
is None and metric
is None:
313 logging
.info('Stopped monitoring vnf: {0}'.format(vnf_name
))
314 return 'Stopped monitoring: {0}'.format(vnf_name
)
316 return 'Error stopping monitoring metric: {0} on {1}:{2}'.format(metric
, vnf_name
, vnf_interface
)
319 # get all metrics defined in the list and export it to Prometheus
320 def get_flow_metrics(self
):
321 while self
.start_monitoring
:
323 self
.monitor_flow_lock
.acquire()
325 for flow_dict
in self
.flow_metrics
:
328 data
['cookie'] = flow_dict
['cookie']
329 data
['cookie_mask'] = COOKIE_MASK
331 if 'tx' in flow_dict
['metric_key']:
332 data
['match'] = {'in_port':flow_dict
['mon_port']}
333 elif 'rx' in flow_dict
['metric_key']:
334 data
['out_port'] = flow_dict
['mon_port']
338 ret
= self
.net
.ryu_REST('stats/flow', dpid
=flow_dict
['switch_dpid'], data
=data
)
339 if isinstance(ret
, dict):
341 elif isinstance(ret
, basestring
):
342 flow_stat_dict
= ast
.literal_eval(ret
.rstrip())
344 flow_stat_dict
= None
346 logging
.debug('received flow stat:{0} '.format(flow_stat_dict
))
348 self
.set_flow_metric(flow_dict
, flow_stat_dict
)
352 if len(self
.flow_metrics
) > 0:
353 pushadd_to_gateway(self
.pushgateway
, job
='sonemu-SDNcontroller', registry
=self
.registry
)
355 logging
.warning("Pushgateway not reachable: {0} {1}".format(Exception, e
))
357 self
.monitor_flow_lock
.release()
360 def get_network_metrics(self
):
361 while self
.start_monitoring
:
363 self
.monitor_lock
.acquire()
365 # group metrics by dpid to optimize the rest api calls
366 dpid_list
= [metric_dict
['switch_dpid'] for metric_dict
in self
.network_metrics
]
367 dpid_set
= set(dpid_list
)
369 for dpid
in dpid_set
:
372 ret
= self
.net
.ryu_REST('stats/port', dpid
=dpid
)
373 if isinstance(ret
, dict):
375 elif isinstance(ret
, basestring
):
376 port_stat_dict
= ast
.literal_eval(ret
.rstrip())
378 port_stat_dict
= None
380 metric_list
= [metric_dict
for metric_dict
in self
.network_metrics
381 if int(metric_dict
['switch_dpid'])==int(dpid
)]
383 for metric_dict
in metric_list
:
384 self
.set_network_metric(metric_dict
, port_stat_dict
)
387 if len(self
.network_metrics
) > 0:
388 pushadd_to_gateway(self
.pushgateway
, job
='sonemu-SDNcontroller', registry
=self
.registry
)
390 logging
.warning("Pushgateway not reachable: {0} {1}".format(Exception, e
))
392 self
.monitor_lock
.release()
395 # add metric to the list to export to Prometheus, parse the Ryu port-stats reply
396 def set_network_metric(self
, metric_dict
, port_stat_dict
):
397 # vnf tx is the datacenter switch rx and vice-versa
398 metric_key
= self
.switch_tx_rx(metric_dict
['metric_key'])
399 switch_dpid
= metric_dict
['switch_dpid']
400 vnf_name
= metric_dict
['vnf_name']
401 vnf_interface
= metric_dict
['vnf_interface']
402 previous_measurement
= metric_dict
['previous_measurement']
403 previous_monitor_time
= metric_dict
['previous_monitor_time']
404 mon_port
= metric_dict
['mon_port']
405 for port_stat
in port_stat_dict
[str(switch_dpid
)]:
406 # ovs output also gives back 'LOCAL' port
407 if port_stat
['port_no'] == 'LOCAL':
409 if int(port_stat
['port_no']) == int(mon_port
):
410 port_uptime
= port_stat
['duration_sec'] + port_stat
['duration_nsec'] * 10 ** (-9)
411 this_measurement
= int(port_stat
[metric_key
])
413 # set prometheus metric
414 self
.prom_metrics
[metric_dict
['metric_key']].\
415 labels(vnf_name
=vnf_name
, vnf_interface
=vnf_interface
, flow_id
=None).\
416 set(this_measurement
)
418 # also the rate is calculated here, but not used for now
419 # (rate can be easily queried from prometheus also)
420 if previous_monitor_time
<= 0 or previous_monitor_time
>= port_uptime
:
421 metric_dict
['previous_measurement'] = int(port_stat
[metric_key
])
422 metric_dict
['previous_monitor_time'] = port_uptime
423 # do first measurement
425 #self.monitor_lock.release()
426 # rate cannot be calculated yet (need a first measurement)
430 time_delta
= (port_uptime
- metric_dict
['previous_monitor_time'])
431 #metric_rate = (this_measurement - metric_dict['previous_measurement']) / float(time_delta)
433 metric_dict
['previous_measurement'] = this_measurement
434 metric_dict
['previous_monitor_time'] = port_uptime
437 logging
.exception('metric {0} not found on {1}:{2}'.format(metric_key
, vnf_name
, vnf_interface
))
438 logging
.exception('monport:{0}, dpid:{1}'.format(mon_port
, switch_dpid
))
439 logging
.exception('monitored network_metrics:{0}'.format(self
.network_metrics
))
440 logging
.exception('port dict:{0}'.format(port_stat_dict
))
441 return 'metric {0} not found on {1}:{2}'.format(metric_key
, vnf_name
, vnf_interface
)
443 def set_flow_metric(self
, metric_dict
, flow_stat_dict
):
444 # vnf tx is the datacenter switch rx and vice-versa
445 metric_key
= metric_dict
['metric_key']
446 switch_dpid
= metric_dict
['switch_dpid']
447 vnf_name
= metric_dict
['vnf_name']
448 vnf_interface
= metric_dict
['vnf_interface']
449 previous_measurement
= metric_dict
['previous_measurement']
450 previous_monitor_time
= metric_dict
['previous_monitor_time']
451 cookie
= metric_dict
['cookie']
454 for flow_stat
in flow_stat_dict
[str(switch_dpid
)]:
455 if 'bytes' in metric_key
:
456 counter
+= flow_stat
['byte_count']
457 elif 'packet' in metric_key
:
458 counter
+= flow_stat
['packet_count']
460 # flow_uptime disabled for now (can give error)
461 #flow_stat = flow_stat_dict[str(switch_dpid)][0]
462 #flow_uptime = flow_stat['duration_sec'] + flow_stat['duration_nsec'] * 10 ** (-9)
464 self
.prom_metrics
[metric_dict
['metric_key']]. \
465 labels(vnf_name
=vnf_name
, vnf_interface
=vnf_interface
, flow_id
=cookie
). \
468 def start_Prometheus(self
, port
=9090):
469 # prometheus.yml configuration file is located in the same directory as this file
473 "-p", "{0}:9090".format(port
),
474 "-v", "{0}/prometheus.yml:/etc/prometheus/prometheus.yml".format(os
.path
.dirname(os
.path
.abspath(__file__
))),
475 "-v", "{0}/profile.rules:/etc/prometheus/profile.rules".format(os
.path
.dirname(os
.path
.abspath(__file__
))),
476 "--name", "prometheus",
479 logging
.info('Start Prometheus container {0}'.format(cmd
))
482 def start_PushGateway(self
, port
=PUSHGATEWAY_PORT
):
486 "-p", "{0}:9091".format(port
),
487 "--name", "pushgateway",
488 "--label", 'com.containernet=""',
492 logging
.info('Start Prometheus Push Gateway container {0}'.format(cmd
))
495 def start_cAdvisor(self
, port
=CADVISOR_PORT
):
499 "--volume=/:/rootfs:ro",
500 "--volume=/var/run:/var/run:rw",
501 "--volume=/sys:/sys:ro",
502 "--volume=/var/lib/docker/:/var/lib/docker:ro",
503 "--publish={0}:8080".format(port
),
505 "--label",'com.containernet=""',
507 "google/cadvisor:latest",
508 #"--storage_duration=1m0s",
509 #"--allow_dynamic_housekeeping=true",
510 #"--housekeeping_interval=1s",
512 logging
.info('Start cAdvisor container {0}'.format(cmd
))
516 # stop the monitoring thread
517 self
.start_monitoring
= False
518 self
.monitor_thread
.join()
519 self
.monitor_flow_thread
.join()
521 # these containers are used for monitoring but are started now outside of son-emu
523 if self
.pushgateway_process
is not None:
524 logging
.info('stopping pushgateway container')
525 self
._stop
_container
('pushgateway')
527 if self
.cadvisor_process
is not None:
528 logging
.info('stopping cadvisor container')
529 self
._stop
_container
('cadvisor')
531 def switch_tx_rx(self
,metric
=''):
532 # when monitoring vnfs, the tx of the datacenter switch is actually the rx of the vnf
533 # so we need to change the metric name to be consistent with the vnf rx or tx
535 metric
= metric
.replace('tx','rx')
537 metric
= metric
.replace('rx','tx')
541 def _stop_container(self
, name
):
543 #container = self.dockercli.containers.get(name)
545 #container.remove(force=True)
547 # the only robust way to stop these containers is via Popen, it seems
549 cmd
= ['docker', 'rm', '-f', name
]
553 def update_skewmon(self
, vnf_name
, resource_name
, action
):
557 config_file_path
= '/tmp/skewmon.cfg'
558 configfile
= open(config_file_path
, 'a+')
560 config
= json
.load(configfile
)
562 #not a valid json file or empty
565 #initialize config file
566 if len(self
.skewmon_metrics
) == 0:
568 json
.dump(config
, configfile
)
571 docker_name
= 'mn.' + vnf_name
572 vnf_container
= self
.dockercli
.containers
.get(docker_name
)
573 key
= resource_name
+ '_' + vnf_container
.short_id
574 vnf_id
= vnf_container
.id
576 if action
== 'start':
577 # add a new vnf to monitor
578 config
[key
] = dict(VNF_NAME
=vnf_name
,
580 VNF_METRIC
=resource_name
)
581 ret
= 'adding to skewness monitor: {0} {1} '.format(vnf_name
, resource_name
)
583 elif action
== 'stop':
584 # remove vnf to monitor
586 ret
= 'removing from skewness monitor: {0} {1} '.format(vnf_name
, resource_name
)
589 self
.skewmon_metrics
= config
590 configfile
= open(config_file_path
, 'w')
591 json
.dump(config
, configfile
)
595 skewmon_container
= self
.dockercli
.containers
.get('skewmon')
597 # remove container if config is empty
599 ret
+= 'stopping skewness monitor'
600 logging
.info('stopping skewness monitor')
601 skewmon_container
.remove(force
=True)
603 except docker
.errors
.NotFound
:
604 # start container if not running
605 ret
+= 'starting skewness monitor'
606 logging
.info('starting skewness monitor')
607 volumes
= {'/sys/fs/cgroup':{'bind':'/sys/fs/cgroup', 'mode':'ro'},
608 '/tmp/skewmon.cfg':{'bind':'/config.txt', 'mode':'ro'}}
609 self
.dockercli
.containers
.run('skewmon',
612 labels
=['com.containernet'],
615 # Wait a while for containers to be completely started
619 list1
= self
.dockercli
.containers
.list(filters
={'status': 'running', 'name': 'prometheus'})
624 return 'skewmon not started'
629 def term(self
, vnf_list
=[]):
631 Start a terminal window for the specified VNFs
632 (start a terminal for all VNFs if vnf_list is empty)
640 if not isinstance(vnf_list
, list):
641 vnf_list
= str(vnf_list
).split(',')
642 vnf_list
= map(str.strip
, vnf_list
)
643 logging
.info('vnf_list: {}'.format(vnf_list
))
645 return self
.start_xterm(vnf_list
)
648 # start an xterm for the specfified vnfs
649 def start_xterm(self
, vnf_names
):
650 # start xterm for all vnfs
651 for vnf_name
in vnf_names
:
652 terminal_cmd
= "docker exec -it mn.{0} /bin/bash".format(vnf_name
)
654 cmd
= ['xterm', '-xrm', 'XTerm*selectToClipboard: true', '-xrm', 'XTerm.vt100.allowTitleOps: false',
659 ret
= 'xterms started for {0}'.format(vnf_names
)
660 if len(vnf_names
) == 0:
661 ret
= 'vnf list is empty, no xterms started'