1 # Copyright (c) 2015 SONATA-NFV and Paderborn University
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # Neither the name of the SONATA-NFV, Paderborn University
17 # nor the names of its contributors may be used to endorse or promote
18 # products derived from this software without specific prior written
21 # This work has been performed in the framework of the SONATA project,
22 # funded by the European Commission under Grant number 671517 through
23 # the Horizon 2020 and 5G-PPP programmes. The authors would like to
24 # acknowledge the contributions of their colleagues of the SONATA
25 # partner consortium (www.sonata-nfv.eu).
27 from mininet
.node
import OVSSwitch
30 from prometheus_client
import Gauge
, CollectorRegistry
, \
31 pushadd_to_gateway
, delete_from_gateway
33 from subprocess
import Popen
37 from copy
import deepcopy
42 class to read openflow stats from the Ryu controller of the DCNetwork
45 PUSHGATEWAY_PORT
= 9091
46 # we cannot use port 8080 because ryu-ofrest api is already using that one
49 COOKIE_MASK
= 0xffffffff
52 class DCNetworkMonitor():
53 def __init__(self
, net
):
55 self
.dockercli
= docker
.from_env()
58 self
.pushgateway
= 'localhost:{0}'.format(PUSHGATEWAY_PORT
)
60 # supported Prometheus metrics
61 self
.registry
= CollectorRegistry()
62 self
.prom_tx_packet_count
= Gauge('sonemu_tx_count_packets', 'Total number of packets sent',
63 ['vnf_name', 'vnf_interface', 'flow_id'], registry
=self
.registry
)
64 self
.prom_rx_packet_count
= Gauge('sonemu_rx_count_packets', 'Total number of packets received',
65 ['vnf_name', 'vnf_interface', 'flow_id'], registry
=self
.registry
)
66 self
.prom_tx_byte_count
= Gauge('sonemu_tx_count_bytes', 'Total number of bytes sent',
67 ['vnf_name', 'vnf_interface', 'flow_id'], registry
=self
.registry
)
68 self
.prom_rx_byte_count
= Gauge('sonemu_rx_count_bytes', 'Total number of bytes received',
69 ['vnf_name', 'vnf_interface', 'flow_id'], registry
=self
.registry
)
71 self
.prom_metrics
= {'tx_packets': self
.prom_tx_packet_count
, 'rx_packets': self
.prom_rx_packet_count
,
72 'tx_bytes': self
.prom_tx_byte_count
, 'rx_bytes': self
.prom_rx_byte_count
}
74 # list of installed metrics to monitor
75 # each entry can contain this data
81 previous_measurement = 0
82 previous_monitor_time = 0
87 self
.monitor_lock
= threading
.Lock()
88 self
.monitor_flow_lock
= threading
.Lock()
89 self
.network_metrics
= []
90 self
.flow_metrics
= []
91 self
.skewmon_metrics
= {}
93 # start monitoring thread
94 self
.start_monitoring
= True
95 self
.monitor_thread
= threading
.Thread(target
=self
.get_network_metrics
)
96 self
.monitor_thread
.start()
98 self
.monitor_flow_thread
= threading
.Thread(
99 target
=self
.get_flow_metrics
)
100 self
.monitor_flow_thread
.start()
103 # cAdvisor, Prometheus pushgateway are started as external container,
104 # to gather monitoring metric in son-emu
105 self
.pushgateway_process
= self
.start_PushGateway()
106 self
.cadvisor_process
= self
.start_cAdvisor()
108 # first set some parameters, before measurement can start
110 def setup_flow(self
, vnf_name
, vnf_interface
=None,
111 metric
='tx_packets', cookie
=0):
115 # check if port is specified (vnf:port)
116 if vnf_interface
is None:
117 # take first interface by default
118 connected_sw
= self
.net
.DCNetwork_graph
.neighbors(vnf_name
)[0]
119 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
120 vnf_interface
= link_dict
[0]['src_port_id']
122 flow_metric
['vnf_name'] = vnf_name
123 flow_metric
['vnf_interface'] = vnf_interface
126 for connected_sw
in self
.net
.DCNetwork_graph
.neighbors(vnf_name
):
127 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
128 for link
in link_dict
:
129 if link_dict
[link
]['src_port_id'] == vnf_interface
:
130 # found the right link and connected switch
131 vnf_switch
= connected_sw
132 flow_metric
['mon_port'] = link_dict
[link
]['dst_port_nr']
136 logging
.exception("vnf switch of {0}:{1} not found!".format(
137 vnf_name
, vnf_interface
))
138 return "vnf switch of {0}:{1} not found!".format(
139 vnf_name
, vnf_interface
)
142 # default port direction to monitor
144 metric
= 'tx_packets'
146 next_node
= self
.net
.getNodeByName(vnf_switch
)
148 if not isinstance(next_node
, OVSSwitch
):
150 "vnf: {0} is not connected to switch".format(vnf_name
))
153 flow_metric
['previous_measurement'] = 0
154 flow_metric
['previous_monitor_time'] = 0
156 flow_metric
['switch_dpid'] = int(str(next_node
.dpid
), 16)
157 flow_metric
['metric_key'] = metric
158 flow_metric
['cookie'] = cookie
160 self
.monitor_flow_lock
.acquire()
161 self
.flow_metrics
.append(flow_metric
)
162 self
.monitor_flow_lock
.release()
164 logging
.info('Started monitoring flow:{3} {2} on {0}:{1}'.format(
165 vnf_name
, vnf_interface
, metric
, cookie
))
166 return 'Started monitoring flow:{3} {2} on {0}:{1}'.format(
167 vnf_name
, vnf_interface
, metric
, cookie
)
169 except Exception as ex
:
170 logging
.exception("setup_metric error.")
173 def stop_flow(self
, vnf_name
, vnf_interface
=None, metric
=None, cookie
=0,):
175 # check if port is specified (vnf:port)
176 if vnf_interface
is None and metric
is not None:
177 # take first interface by default
178 connected_sw
= self
.net
.DCNetwork_graph
.neighbors(vnf_name
)[0]
179 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
180 vnf_interface
= link_dict
[0]['src_port_id']
182 for flow_dict
in self
.flow_metrics
:
183 if flow_dict
['vnf_name'] == vnf_name
and flow_dict
['vnf_interface'] == vnf_interface \
184 and flow_dict
['metric_key'] == metric
and flow_dict
['cookie'] == cookie
:
186 self
.monitor_flow_lock
.acquire()
188 self
.flow_metrics
.remove(flow_dict
)
191 self
.prom_metrics
[flow_dict
['metric_key']]. \
192 labels(vnf_name
=vnf_name
, vnf_interface
=vnf_interface
, flow_id
=cookie
). \
196 self
.pushgateway
, job
='sonemu-SDNcontroller')
198 self
.monitor_flow_lock
.release()
200 logging
.info('Stopped monitoring flow {3}: {2} on {0}:{1}'.format(
201 vnf_name
, vnf_interface
, metric
, cookie
))
202 return 'Stopped monitoring flow {3}: {2} on {0}:{1}'.format(
203 vnf_name
, vnf_interface
, metric
, cookie
)
205 return 'Error stopping monitoring flow: {0} on {1}:{2}'.format(
206 metric
, vnf_name
, vnf_interface
)
208 # first set some parameters, before measurement can start
210 def setup_metric(self
, vnf_name
, vnf_interface
=None, metric
='tx_packets'):
214 # check if port is specified (vnf:port)
215 if vnf_interface
is None or vnf_interface
== '':
216 # take first interface by default
217 connected_sw
= self
.net
.DCNetwork_graph
.neighbors(vnf_name
)[0]
218 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
219 vnf_interface
= link_dict
[0]['src_port_id']
221 network_metric
['vnf_name'] = vnf_name
222 network_metric
['vnf_interface'] = vnf_interface
224 for connected_sw
in self
.net
.DCNetwork_graph
.neighbors(vnf_name
):
225 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
226 for link
in link_dict
:
227 if link_dict
[link
]['src_port_id'] == vnf_interface
:
228 # found the right link and connected switch
229 network_metric
['mon_port'] = link_dict
[link
]['dst_port_nr']
232 if 'mon_port' not in network_metric
:
233 logging
.exception("vnf interface {0}:{1} not found!".format(
234 vnf_name
, vnf_interface
))
235 return "vnf interface {0}:{1} not found!".format(
236 vnf_name
, vnf_interface
)
239 # default port direction to monitor
241 metric
= 'tx_packets'
243 vnf_switch
= self
.net
.DCNetwork_graph
.neighbors(str(vnf_name
))
245 if len(vnf_switch
) > 1:
246 logging
.info("vnf: {0} has multiple ports".format(vnf_name
))
248 elif len(vnf_switch
) == 0:
249 logging
.info("vnf: {0} is not connected".format(vnf_name
))
252 vnf_switch
= vnf_switch
[0]
253 next_node
= self
.net
.getNodeByName(vnf_switch
)
255 if not isinstance(next_node
, OVSSwitch
):
257 "vnf: {0} is not connected to switch".format(vnf_name
))
260 network_metric
['previous_measurement'] = 0
261 network_metric
['previous_monitor_time'] = 0
263 network_metric
['switch_dpid'] = int(str(next_node
.dpid
), 16)
264 network_metric
['metric_key'] = metric
266 self
.monitor_lock
.acquire()
267 self
.network_metrics
.append(network_metric
)
268 self
.monitor_lock
.release()
270 logging
.info('Started monitoring: {2} on {0}:{1}'.format(
271 vnf_name
, vnf_interface
, metric
))
272 return 'Started monitoring: {2} on {0}:{1}'.format(
273 vnf_name
, vnf_interface
, metric
)
275 except Exception as ex
:
276 logging
.exception("setup_metric error.")
279 def stop_metric(self
, vnf_name
, vnf_interface
=None, metric
=None):
281 # check if port is specified (vnf:port)
282 if vnf_interface
is None and metric
is not None:
283 # take first interface by default
284 connected_sw
= self
.net
.DCNetwork_graph
.neighbors(vnf_name
)[0]
285 link_dict
= self
.net
.DCNetwork_graph
[vnf_name
][connected_sw
]
286 vnf_interface
= link_dict
[0]['src_port_id']
288 for metric_dict
in deepcopy(self
.network_metrics
):
289 if metric_dict
['vnf_name'] == vnf_name
and metric_dict
['vnf_interface'] == vnf_interface \
290 and metric_dict
['metric_key'] == metric
:
292 self
.monitor_lock
.acquire()
294 self
.network_metrics
.remove(metric_dict
)
296 # set values to NaN, prometheus api currently does not support removal of metrics
297 # self.prom_metrics[metric_dict['metric_key']].labels(vnf_name, vnf_interface).set(float('nan'))
298 self
.prom_metrics
[metric_dict
['metric_key']]. \
299 labels(vnf_name
=vnf_name
, vnf_interface
=vnf_interface
, flow_id
=None). \
302 # this removes the complete metric, all labels...
303 # 1 single monitor job for all metrics of the SDN controller
304 # we can only remove from the pushgateway grouping keys(labels) which we have defined for the add_to_pushgateway
305 # we can not specify labels from the metrics to be removed
306 # if we need to remove the metrics seperatelty, we need to give
307 # them a separate grouping key, and probably a diffferent
310 self
.pushgateway
, job
='sonemu-SDNcontroller')
312 self
.monitor_lock
.release()
314 logging
.info('Stopped monitoring: {2} on {0}:{1}'.format(
315 vnf_name
, vnf_interface
, metric
))
316 return 'Stopped monitoring: {2} on {0}:{1}'.format(
317 vnf_name
, vnf_interface
, metric
)
319 # delete everything from this vnf
320 elif metric_dict
['vnf_name'] == vnf_name
and vnf_interface
is None and metric
is None:
321 self
.monitor_lock
.acquire()
322 self
.network_metrics
.remove(metric_dict
)
323 logging
.info('remove metric from monitor: vnf_name:{0} vnf_interface:{1} mon_port:{2}'.format(
324 metric_dict
['vnf_name'], metric_dict
['vnf_interface'], metric_dict
['mon_port']))
327 self
.pushgateway
, job
='sonemu-SDNcontroller')
328 self
.monitor_lock
.release()
331 if vnf_interface
is None and metric
is None:
332 logging
.info('Stopped monitoring vnf: {0}'.format(vnf_name
))
333 return 'Stopped monitoring: {0}'.format(vnf_name
)
335 return 'Error stopping monitoring metric: {0} on {1}:{2}'.format(
336 metric
, vnf_name
, vnf_interface
)
339 # get all metrics defined in the list and export it to Prometheus
341 def get_flow_metrics(self
):
342 while self
.start_monitoring
:
344 self
.monitor_flow_lock
.acquire()
346 for flow_dict
in self
.flow_metrics
:
349 data
['cookie'] = flow_dict
['cookie']
350 data
['cookie_mask'] = COOKIE_MASK
352 if 'tx' in flow_dict
['metric_key']:
353 data
['match'] = {'in_port': flow_dict
['mon_port']}
354 elif 'rx' in flow_dict
['metric_key']:
355 data
['out_port'] = flow_dict
['mon_port']
358 ret
= self
.net
.ryu_REST(
359 'stats/flow', dpid
=flow_dict
['switch_dpid'], data
=data
)
360 if isinstance(ret
, dict):
362 elif isinstance(ret
, basestring
):
363 flow_stat_dict
= ast
.literal_eval(ret
.rstrip())
365 flow_stat_dict
= None
367 logging
.debug('received flow stat:{0} '.format(flow_stat_dict
))
369 self
.set_flow_metric(flow_dict
, flow_stat_dict
)
372 if len(self
.flow_metrics
) > 0:
374 self
.pushgateway
, job
='sonemu-SDNcontroller', registry
=self
.registry
)
375 except Exception as e
:
377 "Pushgateway not reachable: {0} {1}".format(Exception, e
))
379 self
.monitor_flow_lock
.release()
382 def get_network_metrics(self
):
383 while self
.start_monitoring
:
385 self
.monitor_lock
.acquire()
387 # group metrics by dpid to optimize the rest api calls
388 dpid_list
= [metric_dict
['switch_dpid']
389 for metric_dict
in self
.network_metrics
]
390 dpid_set
= set(dpid_list
)
392 for dpid
in dpid_set
:
395 ret
= self
.net
.ryu_REST('stats/port', dpid
=dpid
)
396 if isinstance(ret
, dict):
398 elif isinstance(ret
, basestring
):
399 port_stat_dict
= ast
.literal_eval(ret
.rstrip())
401 port_stat_dict
= None
403 metric_list
= [metric_dict
for metric_dict
in self
.network_metrics
404 if int(metric_dict
['switch_dpid']) == int(dpid
)]
406 for metric_dict
in metric_list
:
407 self
.set_network_metric(metric_dict
, port_stat_dict
)
410 if len(self
.network_metrics
) > 0:
412 self
.pushgateway
, job
='sonemu-SDNcontroller', registry
=self
.registry
)
413 except Exception as e
:
415 "Pushgateway not reachable: {0} {1}".format(Exception, e
))
417 self
.monitor_lock
.release()
420 # add metric to the list to export to Prometheus, parse the Ryu port-stats
422 def set_network_metric(self
, metric_dict
, port_stat_dict
):
423 # vnf tx is the datacenter switch rx and vice-versa
424 metric_key
= self
.switch_tx_rx(metric_dict
['metric_key'])
425 switch_dpid
= metric_dict
['switch_dpid']
426 vnf_name
= metric_dict
['vnf_name']
427 vnf_interface
= metric_dict
['vnf_interface']
428 previous_monitor_time
= metric_dict
['previous_monitor_time']
429 mon_port
= metric_dict
['mon_port']
430 for port_stat
in port_stat_dict
[str(switch_dpid
)]:
431 # ovs output also gives back 'LOCAL' port
432 if port_stat
['port_no'] == 'LOCAL':
434 if int(port_stat
['port_no']) == int(mon_port
):
435 port_uptime
= port_stat
['duration_sec'] + \
436 port_stat
['duration_nsec'] * 10 ** (-9)
437 this_measurement
= int(port_stat
[metric_key
])
439 # set prometheus metric
440 self
.prom_metrics
[metric_dict
['metric_key']].\
441 labels(vnf_name
=vnf_name
, vnf_interface
=vnf_interface
, flow_id
=None).\
442 set(this_measurement
)
444 # also the rate is calculated here, but not used for now
445 # (rate can be easily queried from prometheus also)
446 if previous_monitor_time
<= 0 or previous_monitor_time
>= port_uptime
:
447 metric_dict
['previous_measurement'] = int(
448 port_stat
[metric_key
])
449 metric_dict
['previous_monitor_time'] = port_uptime
450 # do first measurement
452 # self.monitor_lock.release()
453 # rate cannot be calculated yet (need a first measurement)
454 metric_dict
['previous_measurement'] = this_measurement
455 metric_dict
['previous_monitor_time'] = port_uptime
458 logging
.exception('metric {0} not found on {1}:{2}'.format(
459 metric_key
, vnf_name
, vnf_interface
))
461 'monport:{0}, dpid:{1}'.format(mon_port
, switch_dpid
))
463 'monitored network_metrics:{0}'.format(self
.network_metrics
))
464 logging
.exception('port dict:{0}'.format(port_stat_dict
))
465 return 'metric {0} not found on {1}:{2}'.format(
466 metric_key
, vnf_name
, vnf_interface
)
468 def set_flow_metric(self
, metric_dict
, flow_stat_dict
):
469 # vnf tx is the datacenter switch rx and vice-versa
470 metric_key
= metric_dict
['metric_key']
471 switch_dpid
= metric_dict
['switch_dpid']
472 vnf_name
= metric_dict
['vnf_name']
473 vnf_interface
= metric_dict
['vnf_interface']
474 cookie
= metric_dict
['cookie']
477 for flow_stat
in flow_stat_dict
[str(switch_dpid
)]:
478 if 'bytes' in metric_key
:
479 counter
+= flow_stat
['byte_count']
480 elif 'packet' in metric_key
:
481 counter
+= flow_stat
['packet_count']
483 # flow_uptime disabled for now (can give error)
484 # flow_stat = flow_stat_dict[str(switch_dpid)][0]
485 # flow_uptime = flow_stat['duration_sec'] + flow_stat['duration_nsec'] * 10 ** (-9)
487 self
.prom_metrics
[metric_dict
['metric_key']]. \
488 labels(vnf_name
=vnf_name
, vnf_interface
=vnf_interface
, flow_id
=cookie
). \
491 def start_Prometheus(self
, port
=9090):
492 # prometheus.yml configuration file is located in the same directory as
497 "-p", "{0}:9090".format(port
),
498 "-v", "{0}/prometheus.yml:/etc/prometheus/prometheus.yml".format(
499 os
.path
.dirname(os
.path
.abspath(__file__
))),
500 "-v", "{0}/profile.rules:/etc/prometheus/profile.rules".format(
501 os
.path
.dirname(os
.path
.abspath(__file__
))),
502 "--name", "prometheus",
505 logging
.info('Start Prometheus container {0}'.format(cmd
))
508 def start_PushGateway(self
, port
=PUSHGATEWAY_PORT
):
512 "-p", "{0}:9091".format(port
),
513 "--name", "pushgateway",
514 "--label", 'com.containernet=""',
518 logging
.info('Start Prometheus Push Gateway container {0}'.format(cmd
))
521 def start_cAdvisor(self
, port
=CADVISOR_PORT
):
525 "--volume=/:/rootfs:ro",
526 "--volume=/var/run:/var/run:rw",
527 "--volume=/sys:/sys:ro",
528 "--volume=/var/lib/docker/:/var/lib/docker:ro",
529 "--publish={0}:8080".format(port
),
531 "--label", 'com.containernet=""',
533 "google/cadvisor:latest",
534 # "--storage_duration=1m0s",
535 # "--allow_dynamic_housekeeping=true",
536 # "--housekeeping_interval=1s",
538 logging
.info('Start cAdvisor container {0}'.format(cmd
))
542 # stop the monitoring thread
543 self
.start_monitoring
= False
544 self
.monitor_thread
.join()
545 self
.monitor_flow_thread
.join()
547 # these containers are used for monitoring but are started now outside
550 if self
.pushgateway_process
is not None:
551 logging
.info('stopping pushgateway container')
552 self
._stop
_container
('pushgateway')
554 if self
.cadvisor_process
is not None:
555 logging
.info('stopping cadvisor container')
556 self
._stop
_container
('cadvisor')
558 def switch_tx_rx(self
, metric
=''):
559 # when monitoring vnfs, the tx of the datacenter switch is actually the rx of the vnf
560 # so we need to change the metric name to be consistent with the vnf rx
563 metric
= metric
.replace('tx', 'rx')
565 metric
= metric
.replace('rx', 'tx')
569 def _stop_container(self
, name
):
571 # container = self.dockercli.containers.get(name)
573 # container.remove(force=True)
575 # the only robust way to stop these containers is via Popen, it seems
577 cmd
= ['docker', 'rm', '-f', name
]
580 def update_skewmon(self
, vnf_name
, resource_name
, action
):
584 config_file_path
= '/tmp/skewmon.cfg'
585 configfile
= open(config_file_path
, 'a+')
587 config
= json
.load(configfile
)
588 except BaseException
:
589 # not a valid json file or empty
592 # initialize config file
593 if len(self
.skewmon_metrics
) == 0:
595 json
.dump(config
, configfile
)
598 docker_name
= 'mn.' + vnf_name
599 vnf_container
= self
.dockercli
.containers
.get(docker_name
)
600 key
= resource_name
+ '_' + vnf_container
.short_id
601 vnf_id
= vnf_container
.id
603 if action
== 'start':
604 # add a new vnf to monitor
605 config
[key
] = dict(VNF_NAME
=vnf_name
,
607 VNF_METRIC
=resource_name
)
608 ret
= 'adding to skewness monitor: {0} {1} '.format(
609 vnf_name
, resource_name
)
611 elif action
== 'stop':
612 # remove vnf to monitor
614 ret
= 'removing from skewness monitor: {0} {1} '.format(
615 vnf_name
, resource_name
)
618 self
.skewmon_metrics
= config
619 configfile
= open(config_file_path
, 'w')
620 json
.dump(config
, configfile
)
624 skewmon_container
= self
.dockercli
.containers
.get('skewmon')
626 # remove container if config is empty
628 ret
+= 'stopping skewness monitor'
629 logging
.info('stopping skewness monitor')
630 skewmon_container
.remove(force
=True)
632 except docker
.errors
.NotFound
:
633 # start container if not running
634 ret
+= 'starting skewness monitor'
635 logging
.info('starting skewness monitor')
636 volumes
= {'/sys/fs/cgroup': {'bind': '/sys/fs/cgroup', 'mode': 'ro'},
637 '/tmp/skewmon.cfg': {'bind': '/config.txt', 'mode': 'ro'}}
638 self
.dockercli
.containers
.run('skewmon',
641 labels
=['com.containernet'],
644 # Wait a while for containers to be completely started
648 list1
= self
.dockercli
.containers
.list(
649 filters
={'status': 'running', 'name': 'prometheus'})
654 return 'skewmon not started'
659 def term(self
, vnf_list
=[]):
661 Start a terminal window for the specified VNFs
662 (start a terminal for all VNFs if vnf_list is empty)
669 if not isinstance(vnf_list
, list):
670 vnf_list
= str(vnf_list
).split(',')
671 vnf_list
= map(str.strip
, vnf_list
)
672 logging
.info('vnf_list: {}'.format(vnf_list
))
674 return self
.start_xterm(vnf_list
)
676 # start an xterm for the specfified vnfs
678 def start_xterm(self
, vnf_names
):
679 # start xterm for all vnfs
680 for vnf_name
in vnf_names
:
681 terminal_cmd
= "docker exec -it mn.{0} /bin/bash".format(vnf_name
)
683 cmd
= ['xterm', '-xrm', 'XTerm*selectToClipboard: true', '-xrm', 'XTerm.vt100.allowTitleOps: false',
688 ret
= 'xterms started for {0}'.format(vnf_names
)
689 if len(vnf_names
) == 0:
690 ret
= 'vnf list is empty, no xterms started'