feat(sol006): sol006 migration

Change-Id: I3db0a79ea89c2340e44fe2b39f00c89e9fb10022
Signed-off-by: bravof <fbravo@whitestack.com>

diff --git a/.gitignore b/.gitignore
index ae5b1ce..aee2396 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -85,6 +85,6 @@ __pycache__/
 !.vscode/launch.json
 !.vscode/extensions.json
 *.code-workspace
 !.vscode/launch.json
 !.vscode/extensions.json
 *.code-workspace

-
+.vscode

 deb_dist
 *.tar.gz
 deb_dist
 *.tar.gz

diff --git a/docker/Dockerfile b/docker/Dockerfile
index a9062c9..e9d355d 100644 (file)
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -20,7 +20,7 @@
 # contact: bdiaz@whitestack.com or glavado@whitestack.com
 ##
 
 # contact: bdiaz@whitestack.com or glavado@whitestack.com
 ##
 

-FROM ubuntu:18.04
+FROM ubuntu:20.04

 
 LABEL authors="Benjamín Díaz"
 
 
 LABEL authors="Benjamín Díaz"
 

diff --git a/docker/scripts/start.sh b/docker/scripts/start.sh
index 0b0cb79..0e50f25 100644 (file)
--- a/docker/scripts/start.sh
+++ b/docker/scripts/start.sh
@@ -20,61 +20,7 @@
 # For those usages not covered by the Apache License, Version 2.0 please
 # contact: glavado@whitestack.com
 ##
 # For those usages not covered by the Apache License, Version 2.0 please
 # contact: glavado@whitestack.com
 ##

-DB_EXISTS=""
-
-max_attempts=120
-function wait_db(){
-    db_host=$1
-    db_port=$2
-    attempt=0
-    echo "Wait until $max_attempts seconds for MySQL mano Server ${db_host}:${db_port} "
-    while ! mysqladmin ping -h"$db_host" -P"$db_port" --silent; do
-        #wait 120 sec
-        if [ $attempt -ge $max_attempts ]; then
-            echo
-            echo "Can not connect to database ${db_host}:${db_port} during $max_attempts sec"
-            return 1
-        fi
-        attempt=$[$attempt+1]
-        echo -n "."
-        sleep 1
-    done
-    return 0
-}
-
-function is_db_created() {
-    db_host=$1
-    db_port=$2
-    db_user=$3
-    db_pswd=$4
-    db_name=$5
-
-    if mysqlshow -h"$db_host" -P"$db_port" -u"$db_user" -p"$db_pswd" | grep -v Wildcard | grep -q $db_name; then
-        echo "DB $db_name exists"
-        return 0
-    else
-        echo "DB $db_name does not exist"
-        return 1
-    fi
-}
-
-if [[ $OSMMON_SQL_DATABASE_URI == *'mysql'* ]]; then
-    DB_HOST=$(echo $OSMMON_SQL_DATABASE_URI | sed -r 's|^.+://.+:.+@(.+):.*$|\1|')
-    DB_PORT=$(echo $OSMMON_SQL_DATABASE_URI | sed -r 's|^.+://.*:([0-9]+).*$|\1|')
-    DB_USER=$(echo $OSMMON_SQL_DATABASE_URI | sed -r 's|^.+://(.+):.+@.+$|\1|')
-    DB_PASSWORD=$(echo $OSMMON_SQL_DATABASE_URI | sed -r 's|^.+://.+:(.+)@.*$|\1|')
-    DB_NAME=$(echo $OSMMON_SQL_DATABASE_URI | sed -r 's|^.+://.+:.+@.+:.*/(\w+)(\?.*)?$|\1|')
-    
-    wait_db "$DB_HOST" "$DB_PORT" || exit 1
-
-    is_db_created "$DB_HOST" "$DB_PORT" "$DB_USER" "$DB_PASSWORD" "$DB_NAME" && DB_EXISTS="Y"
-
-    if [ -z $DB_EXISTS ]; then
-        mysql -h"$DB_HOST" -P"$DB_PORT" -u"$DB_USER" -p"$DB_PASSWORD" --default_character_set utf8 -e "CREATE DATABASE $DB_NAME"
-    fi
-fi

 
 osm-mon-server &
 osm-mon-evaluator &
 
 osm-mon-server &
 osm-mon-evaluator &

-osm-mon-collector &
-osm-mon-dashboarder
\ No newline at end of file
+osm-mon-collector
\ No newline at end of file

diff --git a/osm_mon/collector/infra_collectors/base_osinfra.py b/osm_mon/collector/infra_collectors/base_osinfra.py
index 050e525..1c1999c 100644 (file)
--- a/osm_mon/collector/infra_collectors/base_osinfra.py
+++ b/osm_mon/collector/infra_collectors/base_osinfra.py
@@ -76,7 +76,7 @@ class BaseOpenStackInfraCollector(BaseVimInfraCollector):
                     'nsr_id': nsr_id,
                     'ns_name': ns_name,
                     'vnf_member_index': vnf_member_index,
                     'nsr_id': nsr_id,
                     'ns_name': ns_name,
                     'vnf_member_index': vnf_member_index,

-                    'vdur_name': vdur['name'],
+                    'vdur_name': vdur.get("name", ""),

                     'project_id': vnfr_project_id
                 }
                 try:
                     'project_id': vnfr_project_id
                 }
                 try:

diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py
index 7de93ec..2738d5c 100644 (file)
--- a/osm_mon/collector/vnf_collectors/openstack.py
+++ b/osm_mon/collector/vnf_collectors/openstack.py
@@ -91,7 +91,6 @@ class OpenstackCollector(BaseVimCollector):
         nsr_id = vnfr['nsr-id-ref']
         vnf_member_index = vnfr['member-vnf-index-ref']
         vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])
         nsr_id = vnfr['nsr-id-ref']
         vnf_member_index = vnfr['member-vnf-index-ref']
         vnfd = self.common_db.get_vnfd(vnfr['vnfd-id'])

-

         # Populate extra tags for metrics
         tags = {}
         tags['ns_name'] = self.common_db.get_nsr(nsr_id)['name']
         # Populate extra tags for metrics
         tags = {}
         tags['ns_name'] = self.common_db.get_nsr(nsr_id)['name']


@@ -101,6 +100,7 @@ class OpenstackCollector(BaseVimCollector):
             tags['project_id'] = ''
 
         metrics = []
             tags['project_id'] = ''
 
         metrics = []

+

         for vdur in vnfr['vdur']:
             # This avoids errors when vdur records have not been completely filled
             if 'name' not in vdur:
         for vdur in vnfr['vdur']:
             # This avoids errors when vdur records have not been completely filled
             if 'name' not in vdur:


@@ -108,12 +108,11 @@ class OpenstackCollector(BaseVimCollector):
             vdu = next(
                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
             )
             vdu = next(
                 filter(lambda vdu: vdu['id'] == vdur['vdu-id-ref'], vnfd['vdu'])
             )

-            if 'monitoring-param' in vdu:
-                for param in vdu['monitoring-param']:
-                    metric_name = param['nfvi-metric']
-                    interface_name = param['interface-name-ref'] if 'interface-name-ref' in param else None
+            if 'monitoring-parameter' in vdu:
+                for param in vdu['monitoring-parameter']:
+                    metric_name = param['performance-metric']

                     openstack_metric_name = METRIC_MAPPINGS[metric_name]
                     openstack_metric_name = METRIC_MAPPINGS[metric_name]

-                    metric_type = self._get_metric_type(metric_name, interface_name)
+                    metric_type = self._get_metric_type(metric_name)

                     try:
                         resource_id = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
                     except ValueError:
                     try:
                         resource_id = self._get_resource_uuid(nsr_id, vnf_member_index, vdur['name'])
                     except ValueError:


@@ -123,14 +122,14 @@ class OpenstackCollector(BaseVimCollector):
                             vdur['name'], vnf_member_index, nsr_id)
                         continue
                     try:
                             vdur['name'], vnf_member_index, nsr_id)
                         continue
                     try:

-                        log.info("Collecting metric type: %s and metric_name: %s and resource_id %s and "
-                                 "interface_name: %s", metric_type, metric_name, resource_id, interface_name)
-                        value = self.backend.collect_metric(metric_type, openstack_metric_name, resource_id,
-                                                            interface_name)
+                        log.info(
+                            "Collecting metric type: %s and metric_name: %s and resource_id %s and ",
+                            metric_type,
+                            metric_name,
+                            resource_id)
+                        value = self.backend.collect_metric(metric_type, openstack_metric_name, resource_id)

                         if value is not None:
                             log.info("value: %s", value)
                         if value is not None:
                             log.info("value: %s", value)

-                            if interface_name:
-                                tags['interface'] = interface_name

                             metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], metric_name, value, tags)
                             metrics.append(metric)
                         else:
                             metric = VnfMetric(nsr_id, vnf_member_index, vdur['name'], metric_name, value, tags)
                             metrics.append(metric)
                         else:


@@ -152,17 +151,15 @@ class OpenstackCollector(BaseVimCollector):
             log.info("Using ceilometer backend to collect metric")
             return ceilometer
 
             log.info("Using ceilometer backend to collect metric")
             return ceilometer
 

-    def _get_metric_type(self, metric_name: str, interface_name: str) -> MetricType:
+    def _get_metric_type(self, metric_name: str) -> MetricType:

         if metric_name not in INTERFACE_METRICS:
             return MetricType.INSTANCE
         else:
         if metric_name not in INTERFACE_METRICS:
             return MetricType.INSTANCE
         else:

-            if interface_name:
-                return MetricType.INTERFACE_ONE

             return MetricType.INTERFACE_ALL
 
 
 class OpenstackBackend:
             return MetricType.INTERFACE_ALL
 
 
 class OpenstackBackend:

-    def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str, interface_name: str):
+    def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str):

         pass
 
 
         pass
 
 


@@ -180,10 +177,7 @@ class GnocchiBackend(OpenstackBackend):
         sess = OpenstackUtils.get_session(vim_account)
         return neutron_client.Client(session=sess)
 
         sess = OpenstackUtils.get_session(vim_account)
         return neutron_client.Client(session=sess)
 

-    def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str, interface_name: str):
-        if metric_type == MetricType.INTERFACE_ONE:
-            return self._collect_interface_one_metric(metric_name, resource_id, interface_name)
-
+    def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str):

         if metric_type == MetricType.INTERFACE_ALL:
             return self._collect_interface_all_metric(metric_name, resource_id)
 
         if metric_type == MetricType.INTERFACE_ALL:
             return self._collect_interface_all_metric(metric_name, resource_id)
 


@@ -193,21 +187,6 @@ class GnocchiBackend(OpenstackBackend):
         else:
             raise Exception('Unknown metric type %s' % metric_type.value)
 
         else:
             raise Exception('Unknown metric type %s' % metric_type.value)
 

-    def _collect_interface_one_metric(self, metric_name, resource_id, interface_name):
-        ports = self.neutron.list_ports(name=interface_name, device_id=resource_id)
-        if not ports or not ports['ports']:
-            raise Exception(
-                'Port not found for interface %s on instance %s' % (interface_name, resource_id))
-        port = ports['ports'][0]
-        port_uuid = port['id'][:11]
-        tap_name = 'tap' + port_uuid
-        interfaces = self.client.resource.search(resource_type='instance_network_interface',
-                                                 query={'=': {'name': tap_name}})
-        measures = self.client.metric.get_measures(metric_name,
-                                                   resource_id=interfaces[0]['id'],
-                                                   limit=1)
-        return measures[-1][2] if measures else None
-

     def _collect_interface_all_metric(self, openstack_metric_name, resource_id):
         total_measure = None
         interfaces = self.client.resource.search(resource_type='instance_network_interface',
     def _collect_interface_all_metric(self, openstack_metric_name, resource_id):
         total_measure = None
         interfaces = self.client.resource.search(resource_type='instance_network_interface',


@@ -279,7 +258,7 @@ class CeilometerBackend(OpenstackBackend):
         sess = OpenstackUtils.get_session(vim_account)
         return ceilometer_client.Client("2", session=sess)
 
         sess = OpenstackUtils.get_session(vim_account)
         return ceilometer_client.Client("2", session=sess)
 

-    def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str, interface_name: str):
+    def collect_metric(self, metric_type: MetricType, metric_name: str, resource_id: str):

         if metric_type != MetricType.INSTANCE:
             raise NotImplementedError('Ceilometer backend only support instance metrics')
         measures = self.client.samples.list(meter_name=metric_name, limit=1, q=[
         if metric_type != MetricType.INSTANCE:
             raise NotImplementedError('Ceilometer backend only support instance metrics')
         measures = self.client.samples.list(meter_name=metric_name, limit=1, q=[

diff --git a/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py b/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py
index 1f2d06a..42df03c 100644 (file)
--- a/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py
+++ b/osm_mon/tests/unit/collector/vnf_collectors/test_openstack.py
@@ -41,6 +41,7 @@ class CollectorTest(TestCase):
     @mock.patch.object(GnocchiBackend, '_build_gnocchi_client')
     def test_collect_gnocchi_rate_instance(self, build_gnocchi_client, _):
         mock_gnocchi_client = mock.Mock()
     @mock.patch.object(GnocchiBackend, '_build_gnocchi_client')
     def test_collect_gnocchi_rate_instance(self, build_gnocchi_client, _):
         mock_gnocchi_client = mock.Mock()

+        mock_gnocchi_client.metric = mock.Mock()

         mock_gnocchi_client.metric.get_measures.return_value = [(datetime.datetime(2019, 4, 12, 15, 43,
                                                                                    tzinfo=datetime.timezone(
                                                                                        datetime.timedelta(0),
         mock_gnocchi_client.metric.get_measures.return_value = [(datetime.datetime(2019, 4, 12, 15, 43,
                                                                                    tzinfo=datetime.timezone(
                                                                                        datetime.timedelta(0),


@@ -96,44 +97,6 @@ class CollectorTest(TestCase):
                                                                         start=mock.ANY,
                                                                         resource_id='test_resource_id')
 
                                                                         start=mock.ANY,
                                                                         resource_id='test_resource_id')
 

-    @mock.patch.object(GnocchiBackend, '_build_neutron_client')
-    @mock.patch.object(GnocchiBackend, '_build_gnocchi_client')
-    def test_collect_interface_one_metric(self, build_gnocchi_client, build_neutron_client):
-        mock_gnocchi_client = mock.Mock()
-        mock_gnocchi_client.resource.search.return_value = [{'id': 'test_id'}]
-        mock_gnocchi_client.metric.get_measures.return_value = [(datetime.datetime(2019, 4, 12, 15, 43,
-                                                                                   tzinfo=datetime.timezone(
-                                                                                       datetime.timedelta(0),
-                                                                                       '+00:00')), 60.0, 0.0345442539),
-                                                                (datetime.datetime(2019, 4, 12, 15, 44,
-                                                                                   tzinfo=datetime.timezone(
-                                                                                       datetime.timedelta(0),
-                                                                                       '+00:00')), 60.0, 0.0333070363)]
-        mock_neutron_client = mock.Mock()
-        mock_neutron_client.list_ports.return_value = {'ports': [
-            {'binding:vnic_type': 'normal', 'created_at': '2019-04-15T17:11:39Z',
-             'tenant_id': '88b78c76e01f4b228e68a06f8ebec6da',
-             'binding:vif_details': {'port_filter': True, 'ovs_hybrid_plug': True, 'datapath_type': 'system'},
-             'revision_number': 6, 'updated_at': '2019-04-15T17:11:48Z', 'binding:profile': {},
-             'mac_address': 'fa:16:3e:1c:e3:00', 'status': 'ACTIVE', 'project_id': '88b78c76e01f4b228e68a06f8ebec6da',
-             'network_id': '7f34edad-9064-4de8-b535-b14f9b715ed9', 'port_security_enabled': True, 'tags': [],
-             'description': '', 'security_groups': ['3d60c37e-6229-487b-858d-3aff6d98d66f'], 'admin_state_up': True,
-             'binding:vif_type': 'ovs', 'allowed_address_pairs': [], 'name': 'eth0',
-             'id': '1c6f9a06-6b88-45f3-8d32-dc1216436f0a', 'binding:host_id': 's111412',
-             'fixed_ips': [{'ip_address': '10.0.0.51', 'subnet_id': '4c3fa16c-3e22-43f4-9798-7b10593aff93'}],
-             'device_owner': 'compute:nova', 'device_id': '5cf5bbc4-e4f8-4e22-8bbf-6970218e774d',
-             'extra_dhcp_opts': []}]}
-
-        build_gnocchi_client.return_value = mock_gnocchi_client
-        build_neutron_client.return_value = mock_neutron_client
-
-        backend = GnocchiBackend({'_id': 'test_uuid'})
-        value = backend._collect_interface_one_metric('packets_received', 'test_resource_id', 'eth0')
-        self.assertEqual(value, 0.0333070363)
-        mock_gnocchi_client.metric.get_measures.assert_called_once_with('packets_received', resource_id='test_id',
-                                                                        limit=1)
-        mock_neutron_client.list_ports.assert_called_once_with(device_id='test_resource_id', name='eth0')
-

     @mock.patch.object(GnocchiBackend, '_build_neutron_client')
     @mock.patch.object(GnocchiBackend, '_build_gnocchi_client')
     def test_collect_interface_all_metric(self, build_gnocchi_client, build_neutron_client):
     @mock.patch.object(GnocchiBackend, '_build_neutron_client')
     @mock.patch.object(GnocchiBackend, '_build_gnocchi_client')
     def test_collect_interface_all_metric(self, build_gnocchi_client, build_neutron_client):

diff --git a/requirements.txt b/requirements.txt
index 558f045..eab965e 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,14 +18,17 @@
 
 # For those usages not covered by the Apache License, Version 2.0 please
 # contact: prithiv.mohan@intel.com or adrian.hoban@intel.com
 
 # For those usages not covered by the Apache License, Version 2.0 please
 # contact: prithiv.mohan@intel.com or adrian.hoban@intel.com

+chardet==3.0.4
+lxml==4.6.2
+humanfriendly==9.0.*

 aiokafka==0.6.0
 aiokafka==0.6.0

-requests==2.18.*
+requests==2.25.*

 python-keystoneclient==3.15.*
 python-keystoneclient==3.15.*

-six
+six==1.15.0

 pyyaml>=5.1.2
 prometheus_client==0.4.*
 gnocchiclient==7.0.*
 pyyaml>=5.1.2
 prometheus_client==0.4.*
 gnocchiclient==7.0.*

-pyvcloud==19.1.*
+pyvcloud==23.0.*

 python-ceilometerclient==2.9.*
 python-novaclient==12.0.*
 python-neutronclient==5.1.*
 python-ceilometerclient==2.9.*
 python-novaclient==12.0.*
 python-neutronclient==5.1.*

diff --git a/setup.py b/setup.py
index 485ed51..fe8af5a 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -52,7 +52,7 @@ setup(
     package_dir={_name: _name},
     install_requires=[
         "aiokafka==0.6.0",
     package_dir={_name: _name},
     install_requires=[
         "aiokafka==0.6.0",

-        "requests==2.18.*",
+        "requests==2.25.*",

         "python-keystoneclient==3.15.*",
         "six",
         "pyyaml>=5.1.2",
         "python-keystoneclient==3.15.*",
         "six",
         "pyyaml>=5.1.2",

diff --git a/tox.ini b/tox.ini
index efea4e6..595b5a9 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -25,7 +25,7 @@
 # test suite on all supported python versions. To use it, "pip install tox"
 # and then run "tox" from this directory.
 [tox]
 # test suite on all supported python versions. To use it, "pip install tox"
 # and then run "tox" from this directory.
 [tox]

-envlist = cover, py3, flake8, pylint
+envlist = cover, py3, flake8

 
 
 [testenv]
 
 
 [testenv]


@@ -51,14 +51,6 @@ commands =
   coverage html -d ./cover --omit='*tests*'
   coverage xml -o coverage.xml --omit='*tests*'
 
   coverage html -d ./cover --omit='*tests*'
   coverage xml -o coverage.xml --omit='*tests*'
 

-[testenv:pylint]
-basepython = python3
-deps = pylint
-       -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-commands =
-    pylint -E osm_mon
-

 [testenv:flake8]
 basepython = python3
 deps = flake8
 [testenv:flake8]
 basepython = python3
 deps = flake8