rwlaunchpad/ra/pytest/ns/pingpong/test_scaling.py

   1 #!/usr/bin/env python
   2 """
   3 #
   4 #   Copyright 2016 RIFT.IO Inc
   5 #
   6 #   Licensed under the Apache License, Version 2.0 (the "License");
   7 #   you may not use this file except in compliance with the License.
   8 #   You may obtain a copy of the License at
   9 #
  10 #       http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 #   Unless required by applicable law or agreed to in writing, software
  13 #   distributed under the License is distributed on an "AS IS" BASIS,
  14 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 #   See the License for the specific language governing permissions and
  16 #   limitations under the License.
  17 #
  18
  19 @file test_scaling.py
  20 @author Paul Laidler (Paul.Laidler@riftio.com)
  21 @date 07/13/2016
  22 @brief Pingpong scaling system test
  23 """
  24
  25 import os
  26 import pytest
  27 import subprocess
  28 import sys
  29 import time
  30 import uuid
  31
  32 import rift.auto.mano
  33 import rift.auto.session
  34 import rift.auto.descriptor
  35
  36 from gi.repository import (
  37     NsrYang,
  38     NsdYang,
  39     VnfrYang,
  40     RwNsrYang,
  41     RwNsdYang,
  42     RwVnfrYang,
  43 )
  44
  45 @pytest.mark.setup('pingpong_nsd')
  46 @pytest.mark.depends('launchpad')
  47 class TestSetupPingpongNsd(object):
  48     def test_onboard(self, mgmt_session, descriptors):
  49         for descriptor in descriptors:
  50             rift.auto.descriptor.onboard(mgmt_session.host, descriptor)
  51
  52     def test_install_sar(self, mgmt_session):
  53         install_cmd = 'ssh {mgmt_ip} -q -n -o BatchMode=yes -o StrictHostKeyChecking=no -- sudo yum install sysstat --assumeyes'.format(
  54                 mgmt_ip=mgmt_session.host,
  55         )
  56         subprocess.check_call(install_cmd, shell=True)
  57
  58
  59 @pytest.fixture(scope='function', params=[5,10,15,20,25])
  60 def service_count(request):
  61     '''Fixture representing the number of services to test'''
  62     return request.param
  63
  64 @pytest.mark.depends('pingpong_nsd')
  65 class TestScaling(object):
  66     @pytest.mark.preserve_fixture_order
  67     def test_scaling(self, mgmt_session, cloud_account_name, service_count):
  68
  69         def start_services(mgmt_session, desired_service_count, max_attempts=3):
  70             catalog = mgmt_session.proxy(NsdYang).get_config('/nsd-catalog')
  71             nsd = catalog.nsd[0]
  72
  73             nsr_path = "/ns-instance-config"
  74             nsr = mgmt_session.proxy(RwNsrYang).get_config(nsr_path)
  75             service_count = len(nsr.nsr)
  76
  77             attempts = 0
  78             while attempts < max_attempts and service_count < desired_service_count:
  79                 attempts += 1
  80
  81                 for count in range(service_count, desired_service_count):
  82                     nsr = rift.auto.descriptor.create_nsr(
  83                         cloud_account_name,
  84                         "pingpong_%s" % str(uuid.uuid4().hex[:10]),
  85                         nsd.id)
  86                     mgmt_session.proxy(RwNsrYang).create_config('/ns-instance-config/nsr', nsr)
  87
  88                 ns_instance_opdata = mgmt_session.proxy(RwNsrYang).get('/ns-instance-opdata')
  89                 for nsr in ns_instance_opdata.nsr:
  90                     try:
  91                         xpath = "/ns-instance-opdata/nsr[ns-instance-config-ref='{}']/operational-status".format(nsr.ns_instance_config_ref)
  92                         mgmt_session.proxy(RwNsrYang).wait_for(xpath, "running", fail_on=['failed'], timeout=180)
  93                         xpath = "/ns-instance-opdata/nsr[ns-instance-config-ref='{}']/config-status".format(nsr.ns_instance_config_ref)
  94                         mgmt_session.proxy(RwNsrYang).wait_for(xpath, "configured", fail_on=['failed'], timeout=450)
  95                         service_count += 1
  96                     except rift.auto.session.ProxyWaitForError:
  97                         mgmt_session.proxy(RwNsrYang).delete_config("/ns-instance-config/nsr[id='{}']".format(nsr.ns_instance_config_ref))
  98
  99         def monitor_launchpad_performance(service_count, interval=30, samples=1):
 100             sar_cmd = "ssh {mgmt_ip} -q -n -o BatchMode=yes -o StrictHostKeyChecking=no -- sar -A {interval} {samples}".format(
 101                     mgmt_ip=mgmt_session.host,
 102                     interval=interval,
 103                     samples=samples
 104             )
 105             output = subprocess.check_output(sar_cmd, shell=True, stderr=subprocess.STDOUT)
 106             outfile = '{rift_artifacts}/scaling_{task_id}.log'.format(
 107                     rift_artifacts=os.environ.get('RIFT_ARTIFACTS'),
 108                     task_id=os.environ.get('AUTO_TASK_ID')
 109             )
 110             with open(outfile, 'a') as fh:
 111                 message = '''
 112 == SCALING RESULTS : {service_count} Network Services ==
 113 {output}
 114                 '''.format(service_count=service_count, output=output.decode())
 115                 fh.write(message)
 116
 117         start_services(mgmt_session, service_count)
 118         monitor_launchpad_performance(service_count, interval=30, samples=1)
 119
 120 @pytest.mark.depends('pingpong_nsd')
 121 @pytest.mark.teardown('pingpong_nsd')
 122 class TestTeardownPingpongNsr(object):
 123     def test_teardown_nsr(self, mgmt_session):
 124
 125         ns_instance_config = mgmt_session.proxy(RwNsrYang).get_config('/ns-instance-config')
 126         for nsr in ns_instance_config.nsr:
 127             mgmt_session.proxy(RwNsrYang).delete_config("/ns-instance-config/nsr[id='{}']".format(nsr.id))
 128
 129         time.sleep(60)
 130         vnfr_catalog = mgmt_session.proxy(RwVnfrYang).get('/vnfr-catalog')
 131         assert vnfr_catalog is None or len(vnfr_catalog.vnfr) == 0
 132
 133     def test_generate_plots(self):
 134         plot_commands = [
 135             ('python {rift_install}/usr/rift/systemtest/util/sarplot.py '
 136                     '--plot "{rift_artifacts}/scaling_cpu_{task_id}.png" '
 137                     '--title "CPU Utilization by network service count" '
 138                     '--keys CPU '
 139                     '--fields %usr,%idle,%sys '
 140                     '--key-filter CPU:all '
 141                     '--ylabel "CPU Utilization %" '
 142                     '--xlabel "Network Service Count" '
 143                     '--xticklabels "5,10,15,20,25" < {rift_artifacts}/scaling_{task_id}.log'
 144             ),
 145             ('python {rift_install}/usr/rift/systemtest/util/sarplot.py '
 146                     '--plot "{rift_artifacts}/scaling_mem_{task_id}.png" '
 147                     '--title "Memory Utilization by network service count" '
 148                     '--fields kbmemfree,kbmemused,kbbuffers,kbcached,kbcommit,kbactive,kbinact,kbdirty '
 149                     '--ylabel "Memory Utilization" '
 150                     '--xlabel "Network Service Count" '
 151                     '--xticklabels "5,10,15,20,25" < {rift_artifacts}/scaling_{task_id}.log'
 152             ),
 153             ('python {rift_install}/usr/rift/systemtest/util/sarplot.py '
 154                     '--plot "{rift_artifacts}/scaling_mempct_{task_id}.png" '
 155                     '--title "Memory Utilization by network service count" '
 156                     '--fields %memused,%commit '
 157                     '--ylabel "Memory Utilization %" '
 158                     '--xlabel "Network Service Count" '
 159                     '--xticklabels "5,10,15,20,25" < {rift_artifacts}/scaling_{task_id}.log'
 160             ),
 161             ('python {rift_install}/usr/rift/systemtest/util/sarplot.py '
 162                     '--plot "{rift_artifacts}/scaling_iface_{task_id}.png" '
 163                     '--title "Interface Utilization by network service count" '
 164                     '--keys IFACE '
 165                     '--fields rxpck/s,txpck/s,rxkB/s,txkB/s,rxcmp/s,txcmp/s,rxmcst/s '
 166                     '--key-filter IFACE:eth0 '
 167                     '--ylabel "Interface Utilization" '
 168                     '--xlabel "Network Service Count" '
 169                     '--xticklabels "5,10,15,20,25" < {rift_artifacts}/scaling_{task_id}.log'
 170             ),
 171             ('python {rift_install}/usr/rift/systemtest/util/sarplot.py '
 172                     '--plot "{rift_artifacts}/scaling_iface_err_{task_id}.png" '
 173                     '--title "Interface Errors by network service count" '
 174                     '--keys IFACE '
 175                     '--fields rxerr/s,txerr/s,coll/s,rxdrop/s,txdrop/s,txcarr/s,rxfram/s,rxfifo/s,txfifo/s '
 176                     '--key-filter IFACE:eth0 '
 177                     '--ylabel "Interface Errors" '
 178                     '--xlabel "Network Service Count" '
 179                     '--xticklabels "5,10,15,20,25" < {rift_artifacts}/scaling_{task_id}.log'
 180             ),
 181         ]
 182
 183         for cmd in plot_commands:
 184             subprocess.check_call(
 185                     cmd.format(
 186                         rift_install=os.environ.get('RIFT_INSTALL'),
 187                         rift_artifacts=os.environ.get('RIFT_ARTIFACTS'),
 188                         task_id=os.environ.get('AUTO_TASK_ID')
 189                     ),
 190                     shell=True
 191             )
 192