From: bravof Date: Wed, 2 Dec 2020 19:42:05 +0000 (-0300) Subject: feat(sol006): sol006 migration X-Git-Tag: release-v9.0-start~2 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=refs%2Fchanges%2F76%2F10076%2F6;p=osm%2FMON.git feat(sol006): sol006 migration Change-Id: I3db0a79ea89c2340e44fe2b39f00c89e9fb10022 Signed-off-by: bravof --- diff --git a/.gitignore b/.gitignore index ae5b1ce..aee2396 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,6 @@ __pycache__/ !.vscode/launch.json !.vscode/extensions.json *.code-workspace - +.vscode deb_dist *.tar.gz diff --git a/docker/Dockerfile b/docker/Dockerfile index a9062c9..e9d355d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ # contact: bdiaz@whitestack.com or glavado@whitestack.com ## -FROM ubuntu:18.04 +FROM ubuntu:20.04 LABEL authors="Benjamín Díaz" diff --git a/docker/scripts/start.sh b/docker/scripts/start.sh index 0b0cb79..0e50f25 100644 --- a/docker/scripts/start.sh +++ b/docker/scripts/start.sh @@ -20,61 +20,7 @@ # For those usages not covered by the Apache License, Version 2.0 please # contact: glavado@whitestack.com ## -DB_EXISTS="" - -max_attempts=120 -function wait_db(){ - db_host=$1 - db_port=$2 - attempt=0 - echo "Wait until $max_attempts seconds for MySQL mano Server ${db_host}:${db_port} " - while ! mysqladmin ping -h"$db_host" -P"$db_port" --silent; do - #wait 120 sec - if [ $attempt -ge $max_attempts ]; then - echo - echo "Can not connect to database ${db_host}:${db_port} during $max_attempts sec" - return 1 - fi - attempt=$[$attempt+1] - echo -n "." - sleep 1 - done - return 0 -} - -function is_db_created() { - db_host=$1 - db_port=$2 - db_user=$3 - db_pswd=$4 - db_name=$5 - - if mysqlshow -h"$db_host" -P"$db_port" -u"$db_user" -p"$db_pswd" | grep -v Wildcard | grep -q $db_name; then - echo "DB $db_name exists" - return 0 - else - echo "DB $db_name does not exist" - return 1 - fi -} - -if [[ $OSMMON_SQL_DATABASE_URI == *'mysql'* ]]; then - DB_HOST=$(echo $OSMMON_SQL_DATABASE_URI | sed -r 's|^.+://.+:.+@(.+):.*$|\1|') - DB_PORT=$(echo $OSMMON_SQL_DATABASE_URI | sed -r 's|^.+://.*:([0-9]+).*$|\1|') - DB_USER=$(echo $OSMMON_SQL_DATABASE_URI | sed -r 's|^.+://(.+):.+@.+$|\1|') - DB_PASSWORD=$(echo $OSMMON_SQL_DATABASE_URI | sed -r 's|^.+://.+:(.+)@.*$|\1|') - DB_NAME=$(echo $OSMMON_SQL_DATABASE_URI | sed -r 's|^.+://.+:.+@.+:.*/(\w+)(\?.*)?$|\1|') - - wait_db "$DB_HOST" "$DB_PORT" || exit 1 - - is_db_created "$DB_HOST" "$DB_PORT" "$DB_USER" "$DB_PASSWORD" "$DB_NAME" && DB_EXISTS="Y" - - if [ -z $DB_EXISTS ]; then - mysql -h"$DB_HOST" -P"$DB_PORT" -u"$DB_USER" -p"$DB_PASSWORD" --default_character_set utf8 -e "CREATE DATABASE $DB_NAME" - fi -fi osm-mon-server & osm-mon-evaluator & -osm-mon-collector & -osm-mon-dashboarder \ No newline at end of file +osm-mon-collector \ No newline at end of file diff --git a/osm_mon/collector/infra_collectors/base_osinfra.py b/osm_mon/collector/infra_collectors/base_osinfra.py index 050e525..1c1999c 100644 --- a/osm_mon/collector/infra_collectors/base_osinfra.py +++ b/osm_mon/collector/infra_collectors/base_osinfra.py @@ -76,7 +76,7 @@ class BaseOpenStackInfraCollector(BaseVimInfraCollector): 'nsr_id': nsr_id, 'ns_name': ns_name, 'vnf_member_index': vnf_member_index, - 'vdur_name': vdur['name'], + 'vdur_name': vdur.get("name", ""), 'project_id': vnfr_project_id } try: diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py index 7de93ec..2738d5c 100644 --- a/osm_mon/collector/vnf_collectors/openstack.py +++ b/osm_mon/collector/vnf_collectors/openstack.py @@ -91,7 +91,6 @@ class OpenstackCollector(BaseVimCollector): nsr_id = vnfr['nsr-id-ref'] vnf_member_index = vnfr['member-vnf-index-ref'] vnfd = self.common_db.get_vnfd(vnfr['vnfd-id']) - # Populate extra tags for metrics tags = {} tags['ns_name'] = self.common_db.get_nsr(nsr_id)['name'] @@ -101,6 +100,7 @@ class OpenstackCollector(BaseVimCollector): tags['project_id'] = '' metrics = [] + for vdur in vnfr['vdur']: # This avoids errors when vdur records have not been completely filled if 'name' not in vdur: @@ -108,12 +108,11 @@ class OpenstackCollector(BaseVimCollector): vdu = next( filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu']) ) - if 'monitoring-param' in vdu: - for param in vdu['monitoring-param']: - metric_name = param['nfvi-metric'] - interface_name = param['interface-name-ref'] if 'interface-name-ref' in param else None + if 'monitoring-parameter' in vdu: + for param in vdu['monitoring-parameter']: + metric_name = param['performance-metric'] openstack_metric_name = METRIC_MAPPINGS[metric_name] - metric_type = self._get_metric_type(metric_name, interface_name) + metric_type = self._get_metric_type(metric_name) try: resource_id = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name']) except ValueError: @@ -123,14 +122,14 @@ class OpenstackCollector(BaseVimCollector): vdur['name'], vnf_member_index, nsr_id) continue try: - log.info("Collecting metric type: %s and metric_name: %s and resource_id %s and " - "interface_name: %s", metric_type, metric_name, resource_id, interface_name) - value = self.backend.collect_metric(metric_type, openstack_metric_name, resource_id, - interface_name) + log.info( + "Collecting metric type: %s and metric_name: %s and resource_id %s and ", + metric_type, + metric_name, + resource_id) + value = self.backend.collect_metric(metric_type, openstack_metric_name, resource_id) if value is not None: log.info("value: %s", value) - if interface_name: - tags['interface'] = interface_name metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], metric_name, value, tags) metrics.append(metric) else: @@ -152,17 +151,15 @@ class OpenstackCollector(BaseVimCollector): log.info("Using ceilometer backend to collect metric") return ceilometer - def _get_metric_type(self, metric_name: str, interface_name: str) -> MetricType: + def _get_metric_type(self, metric_name: str) -> MetricType: if metric_name not in INTERFACE_METRICS: return MetricType.INSTANCE else: - if interface_name: - return MetricType.INTERFACE_ONE return MetricType.INTERFACE_ALL class OpenstackBackend: - def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str, interface_name: str): + def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str): pass @@ -180,10 +177,7 @@ class GnocchiBackend(OpenstackBackend): sess = OpenstackUtils.get_session(vim_account) return neutron_client.Client(session=sess) - def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str, interface_name: str): - if metric_type == MetricType.INTERFACE_ONE: - return self._collect_interface_one_metric(metric_name, resource_id, interface_name) - + def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str): if metric_type == MetricType.INTERFACE_ALL: return self._collect_interface_all_metric(metric_name, resource_id) @@ -193,21 +187,6 @@ class GnocchiBackend(OpenstackBackend): else: raise Exception('Unknown metric type %s' % metric_type.value) - def _collect_interface_one_metric(self, metric_name, resource_id, interface_name): - ports = self.neutron.list_ports(name=interface_name, device_id=resource_id) - if not ports or not ports['ports']: - raise Exception( - 'Port not found for interface %s on instance %s' % (interface_name, resource_id)) - port = ports['ports'][0] - port_uuid = port['id'][:11] - tap_name = 'tap' + port_uuid - interfaces = self.client.resource.search(resource_type='instance_network_interface', - query={'=': {'name': tap_name}}) - measures = self.client.metric.get_measures(metric_name, - resource_id=interfaces[0]['id'], - limit=1) - return measures[-1][2] if measures else None - def _collect_interface_all_metric(self, openstack_metric_name, resource_id): total_measure = None interfaces = self.client.resource.search(resource_type='instance_network_interface', @@ -279,7 +258,7 @@ class CeilometerBackend(OpenstackBackend): sess = OpenstackUtils.get_session(vim_account) return ceilometer_client.Client("2", session=sess) - def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str, interface_name: str): + def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str): if metric_type != MetricType.INSTANCE: raise NotImplementedError('Ceilometer backend only support instance metrics') measures = self.client.samples.list(meter_name=metric_name, limit=1, q=[ diff --git a/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py b/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py index 1f2d06a..42df03c 100644 --- a/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py +++ b/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py @@ -41,6 +41,7 @@ class CollectorTest(TestCase): @mock.patch.object(GnocchiBackend, '_build_gnocchi_client') def test_collect_gnocchi_rate_instance(self, build_gnocchi_client, _): mock_gnocchi_client = mock.Mock() + mock_gnocchi_client.metric = mock.Mock() mock_gnocchi_client.metric.get_measures.return_value = [(datetime.datetime(2019, 4, 12, 15, 43, tzinfo=datetime.timezone( datetime.timedelta(0), @@ -96,44 +97,6 @@ class CollectorTest(TestCase): start=mock.ANY, resource_id='test_resource_id') - @mock.patch.object(GnocchiBackend, '_build_neutron_client') - @mock.patch.object(GnocchiBackend, '_build_gnocchi_client') - def test_collect_interface_one_metric(self, build_gnocchi_client, build_neutron_client): - mock_gnocchi_client = mock.Mock() - mock_gnocchi_client.resource.search.return_value = [{'id': 'test_id'}] - mock_gnocchi_client.metric.get_measures.return_value = [(datetime.datetime(2019, 4, 12, 15, 43, - tzinfo=datetime.timezone( - datetime.timedelta(0), - '+00:00')), 60.0, 0.0345442539), - (datetime.datetime(2019, 4, 12, 15, 44, - tzinfo=datetime.timezone( - datetime.timedelta(0), - '+00:00')), 60.0, 0.0333070363)] - mock_neutron_client = mock.Mock() - mock_neutron_client.list_ports.return_value = {'ports': [ - {'binding:vnic_type': 'normal', 'created_at': '2019-04-15T17:11:39Z', - 'tenant_id': '88b78c76e01f4b228e68a06f8ebec6da', - 'binding:vif_details': {'port_filter': True, 'ovs_hybrid_plug': True, 'datapath_type': 'system'}, - 'revision_number': 6, 'updated_at': '2019-04-15T17:11:48Z', 'binding:profile': {}, - 'mac_address': 'fa:16:3e:1c:e3:00', 'status': 'ACTIVE', 'project_id': '88b78c76e01f4b228e68a06f8ebec6da', - 'network_id': '7f34edad-9064-4de8-b535-b14f9b715ed9', 'port_security_enabled': True, 'tags': [], - 'description': '', 'security_groups': ['3d60c37e-6229-487b-858d-3aff6d98d66f'], 'admin_state_up': True, - 'binding:vif_type': 'ovs', 'allowed_address_pairs': [], 'name': 'eth0', - 'id': '1c6f9a06-6b88-45f3-8d32-dc1216436f0a', 'binding:host_id': 's111412', - 'fixed_ips': [{'ip_address': '10.0.0.51', 'subnet_id': '4c3fa16c-3e22-43f4-9798-7b10593aff93'}], - 'device_owner': 'compute:nova', 'device_id': '5cf5bbc4-e4f8-4e22-8bbf-6970218e774d', - 'extra_dhcp_opts': []}]} - - build_gnocchi_client.return_value = mock_gnocchi_client - build_neutron_client.return_value = mock_neutron_client - - backend = GnocchiBackend({'_id': 'test_uuid'}) - value = backend._collect_interface_one_metric('packets_received', 'test_resource_id', 'eth0') - self.assertEqual(value, 0.0333070363) - mock_gnocchi_client.metric.get_measures.assert_called_once_with('packets_received', resource_id='test_id', - limit=1) - mock_neutron_client.list_ports.assert_called_once_with(device_id='test_resource_id', name='eth0') - @mock.patch.object(GnocchiBackend, '_build_neutron_client') @mock.patch.object(GnocchiBackend, '_build_gnocchi_client') def test_collect_interface_all_metric(self, build_gnocchi_client, build_neutron_client): diff --git a/requirements.txt b/requirements.txt index 558f045..eab965e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,14 +18,17 @@ # For those usages not covered by the Apache License, Version 2.0 please # contact: prithiv.mohan@intel.com or adrian.hoban@intel.com +chardet==3.0.4 +lxml==4.6.2 +humanfriendly==9.0.* aiokafka==0.6.0 -requests==2.18.* +requests==2.25.* python-keystoneclient==3.15.* -six +six==1.15.0 pyyaml>=5.1.2 prometheus_client==0.4.* gnocchiclient==7.0.* -pyvcloud==19.1.* +pyvcloud==23.0.* python-ceilometerclient==2.9.* python-novaclient==12.0.* python-neutronclient==5.1.* diff --git a/setup.py b/setup.py index 485ed51..fe8af5a 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ setup( package_dir={_name: _name}, install_requires=[ "aiokafka==0.6.0", - "requests==2.18.*", + "requests==2.25.*", "python-keystoneclient==3.15.*", "six", "pyyaml>=5.1.2", diff --git a/tox.ini b/tox.ini index efea4e6..595b5a9 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. [tox] -envlist = cover, py3, flake8, pylint +envlist = cover, py3, flake8 [testenv] @@ -51,14 +51,6 @@ commands = coverage html -d ./cover --omit='*tests*' coverage xml -o coverage.xml --omit='*tests*' -[testenv:pylint] -basepython = python3 -deps = pylint - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -commands = - pylint -E osm_mon - [testenv:flake8] basepython = python3 deps = flake8