From 6683456ee66a599d2685832edbde82e66647b223 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Thu, 10 Nov 2022 14:07:04 +0100 Subject: [PATCH] Feature 10964 Airflow monitoring pipeline for VM status and NS topology Change-Id: I61b8abffd54fa87266dca03ac04b5e52b358d85d Signed-off-by: garciadeblas --- Dockerfile | 2 +- devops-stages/stage-archive.sh | 4 +- devops-stages/stage-build.sh | 4 +- devops-stages/stage-test.sh | 6 +- requirements-dev.in | 20 ++ requirements-dev.txt | 366 ++++++++++++++++++++++++ requirements-test.in | 20 ++ requirements-test.txt | 22 ++ requirements.in | 26 ++ requirements.txt | 210 ++++++++++++++ src/dags/.airflowignore | 1 + src/dags/__init__.py | 0 src/dags/multivim_vm_status.py | 166 +++++++++++ src/dags/ns_topology.py | 156 ++++++++++ src/osm_mon/__init__.py | 0 src/osm_mon/core/__init__.py | 0 src/osm_mon/core/common_db.py | 90 ++++++ src/osm_mon/core/config.py | 68 +++++ src/osm_mon/core/config.yaml | 27 ++ src/osm_mon/vim_connectors/__init__.py | 0 src/osm_mon/vim_connectors/azure.py | 183 ++++++++++++ src/osm_mon/vim_connectors/base_vim.py | 26 ++ src/osm_mon/vim_connectors/gcp.py | 92 ++++++ src/osm_mon/vim_connectors/openstack.py | 83 ++++++ tox.ini | 126 ++++++++ 25 files changed, 1694 insertions(+), 4 deletions(-) create mode 100644 requirements-dev.in create mode 100644 requirements-dev.txt create mode 100644 requirements-test.in create mode 100644 requirements-test.txt create mode 100644 requirements.in create mode 100644 requirements.txt create mode 100644 src/dags/.airflowignore create mode 100644 src/dags/__init__.py create mode 100644 src/dags/multivim_vm_status.py create mode 100644 src/dags/ns_topology.py create mode 100644 src/osm_mon/__init__.py create mode 100644 src/osm_mon/core/__init__.py create mode 100644 src/osm_mon/core/common_db.py create mode 100644 src/osm_mon/core/config.py create mode 100644 src/osm_mon/core/config.yaml create mode 100644 src/osm_mon/vim_connectors/__init__.py create mode 100644 src/osm_mon/vim_connectors/azure.py create mode 100644 src/osm_mon/vim_connectors/base_vim.py create mode 100644 src/osm_mon/vim_connectors/gcp.py create mode 100644 src/osm_mon/vim_connectors/openstack.py create mode 100644 tox.ini diff --git a/Dockerfile b/Dockerfile index f2d0e6e..db2155d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ python3-dev \ python3-setuptools -RUN python3 -m easy_install pip==21.3.1 +RUN python3 -m easy_install pip==22.3 RUN pip install tox==3.24.5 diff --git a/devops-stages/stage-archive.sh b/devops-stages/stage-archive.sh index d059670..7c88360 100755 --- a/devops-stages/stage-archive.sh +++ b/devops-stages/stage-archive.sh @@ -14,4 +14,6 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -echo "ARCHIVE" +# + +echo "Nothing to be done" diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index 94949c4..7c88360 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -14,4 +14,6 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -echo "BUILD" +# + +echo "Nothing to be done" diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index 24156af..49aee68 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -14,4 +14,8 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -echo "TEST" +# + +echo "Launching tox" +TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto + diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..ed3f514 --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,20 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +git+https://osm.etsi.org/gerrit/osm/common.git@master#egg=osm-common +-r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master +apache-airflow==2.4.* diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..c848d9e --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,366 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +aiokafka==0.7.2 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master +alembic==1.8.1 + # via apache-airflow +anyio==3.6.2 + # via httpcore +apache-airflow==2.4.3 + # via -r requirements-dev.in +apache-airflow-providers-common-sql==1.3.0 + # via + # apache-airflow + # apache-airflow-providers-sqlite +apache-airflow-providers-ftp==3.2.0 + # via apache-airflow +apache-airflow-providers-http==4.1.0 + # via apache-airflow +apache-airflow-providers-imap==3.1.0 + # via apache-airflow +apache-airflow-providers-sqlite==3.3.0 + # via apache-airflow +apispec[yaml]==3.3.2 + # via flask-appbuilder +argcomplete==2.0.0 + # via apache-airflow +attrs==22.1.0 + # via + # apache-airflow + # cattrs + # jsonschema +babel==2.11.0 + # via flask-babel +blinker==1.5 + # via apache-airflow +cachelib==0.9.0 + # via + # flask-caching + # flask-session +cattrs==22.2.0 + # via apache-airflow +certifi==2022.9.24 + # via + # httpcore + # httpx + # requests +cffi==1.15.1 + # via cryptography +charset-normalizer==2.1.1 + # via requests +click==8.1.3 + # via + # clickclick + # flask + # flask-appbuilder +clickclick==20.10.2 + # via connexion +colorama==0.4.6 + # via flask-appbuilder +colorlog==4.8.0 + # via apache-airflow +commonmark==0.9.1 + # via rich +configupdater==3.1.1 + # via apache-airflow +connexion[flask,swagger-ui]==2.14.1 + # via apache-airflow +cron-descriptor==1.2.32 + # via apache-airflow +croniter==1.3.8 + # via apache-airflow +cryptography==38.0.3 + # via apache-airflow +dataclasses==0.6 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master +deprecated==1.2.13 + # via apache-airflow +dill==0.3.6 + # via apache-airflow +dnspython==2.2.1 + # via email-validator +docutils==0.19 + # via python-daemon +email-validator==1.3.0 + # via flask-appbuilder +exceptiongroup==1.0.4 + # via cattrs +flask==2.2.2 + # via + # apache-airflow + # connexion + # flask-appbuilder + # flask-babel + # flask-caching + # flask-jwt-extended + # flask-login + # flask-session + # flask-sqlalchemy + # flask-wtf +flask-appbuilder==4.1.4 + # via apache-airflow +flask-babel==2.0.0 + # via flask-appbuilder +flask-caching==2.0.1 + # via apache-airflow +flask-jwt-extended==4.4.4 + # via flask-appbuilder +flask-login==0.6.2 + # via + # apache-airflow + # flask-appbuilder +flask-session==0.4.0 + # via apache-airflow +flask-sqlalchemy==2.5.1 + # via flask-appbuilder +flask-wtf==1.0.1 + # via + # apache-airflow + # flask-appbuilder +graphviz==0.20.1 + # via apache-airflow +greenlet==2.0.1 + # via sqlalchemy +gunicorn==20.1.0 + # via apache-airflow +h11==0.14.0 + # via httpcore +httpcore==0.16.1 + # via httpx +httpx==0.23.1 + # via apache-airflow +idna==3.4 + # via + # anyio + # email-validator + # requests + # rfc3986 +importlib-metadata==5.0.0 + # via + # alembic + # apache-airflow + # flask + # markdown +importlib-resources==5.10.0 + # via + # alembic + # apache-airflow + # jsonschema +inflection==0.5.1 + # via connexion +itsdangerous==2.1.2 + # via + # apache-airflow + # connexion + # flask + # flask-wtf +jinja2==3.1.2 + # via + # apache-airflow + # flask + # flask-babel + # python-nvd3 + # swagger-ui-bundle +jsonschema==4.17.0 + # via + # apache-airflow + # connexion + # flask-appbuilder +kafka-python==2.0.2 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master + # aiokafka +lazy-object-proxy==1.8.0 + # via apache-airflow +linkify-it-py==2.0.0 + # via apache-airflow +lockfile==0.12.2 + # via + # apache-airflow + # python-daemon +mako==1.2.4 + # via alembic +markdown==3.4.1 + # via apache-airflow +markdown-it-py==2.1.0 + # via + # apache-airflow + # mdit-py-plugins +markupsafe==2.1.1 + # via + # apache-airflow + # jinja2 + # mako + # werkzeug + # wtforms +marshmallow==3.19.0 + # via + # flask-appbuilder + # marshmallow-enum + # marshmallow-oneofschema + # marshmallow-sqlalchemy +marshmallow-enum==1.5.1 + # via flask-appbuilder +marshmallow-oneofschema==3.0.1 + # via apache-airflow +marshmallow-sqlalchemy==0.26.1 + # via flask-appbuilder +mdit-py-plugins==0.3.1 + # via apache-airflow +mdurl==0.1.2 + # via markdown-it-py +osm-common @ git+https://osm.etsi.org/gerrit/osm/common.git@master + # via -r requirements-dev.in +packaging==21.3 + # via + # apache-airflow + # connexion + # marshmallow +pathspec==0.9.0 + # via apache-airflow +pendulum==2.1.2 + # via apache-airflow +pkgutil-resolve-name==1.3.10 + # via jsonschema +pluggy==1.0.0 + # via apache-airflow +prison==0.2.1 + # via flask-appbuilder +psutil==5.9.4 + # via apache-airflow +pycparser==2.21 + # via cffi +pycrypto==2.6.1 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master +pygments==2.13.0 + # via + # apache-airflow + # rich +pyjwt==2.6.0 + # via + # apache-airflow + # flask-appbuilder + # flask-jwt-extended +pymongo==3.12.3 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master +pyparsing==3.0.9 + # via packaging +pyrsistent==0.19.2 + # via jsonschema +python-daemon==2.3.2 + # via apache-airflow +python-dateutil==2.8.2 + # via + # apache-airflow + # croniter + # flask-appbuilder + # pendulum +python-nvd3==0.15.0 + # via apache-airflow +python-slugify==7.0.0 + # via + # apache-airflow + # python-nvd3 +pytz==2022.6 + # via + # babel + # flask-babel +pytzdata==2020.1 + # via pendulum +pyyaml==5.4.1 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master + # apispec + # clickclick + # connexion +requests==2.28.1 + # via + # apache-airflow-providers-http + # connexion + # requests-toolbelt +requests-toolbelt==0.10.1 + # via apache-airflow-providers-http +rfc3986[idna2008]==1.5.0 + # via httpx +rich==12.6.0 + # via apache-airflow +setproctitle==1.3.2 + # via apache-airflow +six==1.16.0 + # via + # prison + # python-dateutil +sniffio==1.3.0 + # via + # anyio + # httpcore + # httpx +sqlalchemy==1.4.44 + # via + # alembic + # apache-airflow + # flask-appbuilder + # flask-sqlalchemy + # marshmallow-sqlalchemy + # sqlalchemy-jsonfield + # sqlalchemy-utils +sqlalchemy-jsonfield==1.0.0 + # via apache-airflow +sqlalchemy-utils==0.38.3 + # via flask-appbuilder +sqlparse==0.4.3 + # via apache-airflow-providers-common-sql +swagger-ui-bundle==0.0.9 + # via connexion +tabulate==0.9.0 + # via apache-airflow +tenacity==8.1.0 + # via apache-airflow +termcolor==2.1.1 + # via apache-airflow +text-unidecode==1.3 + # via python-slugify +typing-extensions==4.4.0 + # via + # apache-airflow + # rich +uc-micro-py==1.0.1 + # via linkify-it-py +unicodecsv==0.14.1 + # via apache-airflow +urllib3==1.26.12 + # via requests +werkzeug==2.2.2 + # via + # apache-airflow + # connexion + # flask + # flask-jwt-extended + # flask-login +wrapt==1.14.1 + # via deprecated +wtforms==3.0.1 + # via + # flask-appbuilder + # flask-wtf +zipp==3.10.0 + # via + # importlib-metadata + # importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements-test.in b/requirements-test.in new file mode 100644 index 0000000..b8caeb0 --- /dev/null +++ b/requirements-test.in @@ -0,0 +1,20 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +coverage +mock +nose2 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..a7f0e06 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,22 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +coverage==6.5.0 + # via -r requirements-test.in +mock==4.0.3 + # via -r requirements-test.in +nose2==0.12.0 + # via -r requirements-test.in diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..6d8f0a8 --- /dev/null +++ b/requirements.in @@ -0,0 +1,26 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +azure-common +azure-identity +azure-mgmt-compute +google-api-python-client +google-auth +prometheus-client +python-keystoneclient +python-novaclient +pyyaml==5.4.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1e6c775 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,210 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +azure-common==1.1.28 + # via + # -r requirements.in + # azure-mgmt-compute +azure-core==1.26.1 + # via + # azure-identity + # azure-mgmt-core + # msrest +azure-identity==1.12.0 + # via -r requirements.in +azure-mgmt-compute==29.0.0 + # via -r requirements.in +azure-mgmt-core==1.3.2 + # via azure-mgmt-compute +cachetools==5.2.0 + # via google-auth +certifi==2022.9.24 + # via + # msrest + # requests +cffi==1.15.1 + # via cryptography +charset-normalizer==2.1.1 + # via requests +cryptography==38.0.3 + # via + # azure-identity + # msal + # pyjwt +debtcollector==2.5.0 + # via + # oslo-config + # oslo-utils + # python-keystoneclient +google-api-core==2.10.2 + # via google-api-python-client +google-api-python-client==2.66.0 + # via -r requirements.in +google-auth==2.14.1 + # via + # -r requirements.in + # google-api-core + # google-api-python-client + # google-auth-httplib2 +google-auth-httplib2==0.1.0 + # via google-api-python-client +googleapis-common-protos==1.57.0 + # via google-api-core +httplib2==0.21.0 + # via + # google-api-python-client + # google-auth-httplib2 +idna==3.4 + # via requests +iso8601==1.1.0 + # via + # keystoneauth1 + # oslo-utils + # python-novaclient +isodate==0.6.1 + # via msrest +keystoneauth1==5.0.0 + # via + # python-keystoneclient + # python-novaclient +msal==1.20.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity +msgpack==1.0.4 + # via oslo-serialization +msrest==0.7.1 + # via azure-mgmt-compute +netaddr==0.8.0 + # via + # oslo-config + # oslo-utils +netifaces==0.11.0 + # via oslo-utils +oauthlib==3.2.2 + # via requests-oauthlib +os-service-types==1.7.0 + # via keystoneauth1 +oslo-config==9.0.0 + # via python-keystoneclient +oslo-i18n==5.1.0 + # via + # oslo-config + # oslo-utils + # python-keystoneclient + # python-novaclient +oslo-serialization==5.0.0 + # via + # python-keystoneclient + # python-novaclient +oslo-utils==6.1.0 + # via + # oslo-serialization + # python-keystoneclient + # python-novaclient +packaging==21.3 + # via + # oslo-utils + # python-keystoneclient +pbr==5.11.0 + # via + # keystoneauth1 + # os-service-types + # oslo-i18n + # oslo-serialization + # python-keystoneclient + # python-novaclient + # stevedore +portalocker==2.6.0 + # via msal-extensions +prettytable==3.5.0 + # via python-novaclient +prometheus-client==0.15.0 + # via -r requirements.in +protobuf==4.21.9 + # via + # google-api-core + # googleapis-common-protos +pyasn1==0.4.8 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 + # via google-auth +pycparser==2.21 + # via cffi +pyjwt[crypto]==2.6.0 + # via msal +pyparsing==3.0.9 + # via + # httplib2 + # oslo-utils + # packaging +python-keystoneclient==5.0.1 + # via -r requirements.in +python-novaclient==18.2.0 + # via -r requirements.in +pytz==2022.6 + # via + # oslo-serialization + # oslo-utils +pyyaml==5.4.1 + # via + # -r requirements.in + # oslo-config +requests==2.28.1 + # via + # azure-core + # google-api-core + # keystoneauth1 + # msal + # msrest + # oslo-config + # python-keystoneclient + # requests-oauthlib +requests-oauthlib==1.3.1 + # via msrest +rfc3986==2.0.0 + # via oslo-config +rsa==4.9 + # via google-auth +six==1.16.0 + # via + # azure-core + # azure-identity + # google-auth + # google-auth-httplib2 + # isodate + # keystoneauth1 + # python-keystoneclient +stevedore==4.1.1 + # via + # keystoneauth1 + # oslo-config + # python-keystoneclient + # python-novaclient +typing-extensions==4.4.0 + # via azure-core +uritemplate==4.1.1 + # via google-api-python-client +urllib3==1.26.12 + # via requests +wcwidth==0.2.5 + # via prettytable +wrapt==1.14.1 + # via debtcollector diff --git a/src/dags/.airflowignore b/src/dags/.airflowignore new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/dags/.airflowignore @@ -0,0 +1 @@ + diff --git a/src/dags/__init__.py b/src/dags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dags/multivim_vm_status.py b/src/dags/multivim_vm_status.py new file mode 100644 index 0000000..a189112 --- /dev/null +++ b/src/dags/multivim_vm_status.py @@ -0,0 +1,166 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.decorators import task +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +from osm_mon.vim_connectors.azure import AzureCollector +from osm_mon.vim_connectors.gcp import GcpCollector +from osm_mon.vim_connectors.openstack import OpenStackCollector +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway + + +SUPPORTED_VIM_TYPES = ["openstack", "vio", "gcp", "azure"] +PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091" +PROMETHEUS_JOB_PREFIX = "airflow_osm_vm_status_" +PROMETHEUS_METRIC = "vm_status" +PROMETHEUS_METRIC_DESCRIPTION = "VM Status from VIM" +SCHEDULE_INTERVAL = 1 + + +def get_all_vim(): + """Get VIMs from MongoDB""" + print("Getting VIM list") + + cfg = Config() + print(cfg.conf) + common_db = CommonDbClient(cfg) + vim_accounts = common_db.get_vim_accounts() + vim_list = [] + for vim in vim_accounts: + print(f'Read VIM {vim["_id"]} ({vim["name"]})') + vim_list.append( + {"_id": vim["_id"], "name": vim["name"], "vim_type": vim["vim_type"]} + ) + + print(vim_list) + print("Getting VIM list OK") + return vim_list + + +def create_dag(dag_id, dag_number, dag_description, vim_id): + dag = DAG( + dag_id, + catchup=False, + default_args={ + "depends_on_past": False, + "retries": 1, + # "retry_delay": timedelta(minutes=1), + "retry_delay": timedelta(seconds=10), + }, + description=dag_description, + is_paused_upon_creation=False, + # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL), + schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *", + start_date=datetime(2022, 1, 1), + tags=["osm", "vim"], + ) + + with dag: + + def get_vim_collector(vim_account): + """Return a VIM collector for the vim_account""" + vim_type = vim_account["vim_type"] + if "config" in vim_account and "vim_type" in vim_account["config"]: + vim_type = vim_account["config"]["vim_type"].lower() + if vim_type == "vio" and "vrops_site" not in vim_account["config"]: + vim_type = "openstack" + if vim_type == "openstack": + return OpenStackCollector(vim_account) + if vim_type == "gcp": + return GcpCollector(vim_account) + if vim_type == "azure": + return AzureCollector(vim_account) + print(f"VIM type '{vim_type}' not supported") + return None + + def get_all_vm_status(vim_account): + """Get VM status from the VIM""" + collector = get_vim_collector(vim_account) + if collector: + # status = collector.is_vim_ok() + # print(f"VIM status: {status}") + vm_status_list = collector.collect_servers_status() + return vm_status_list + else: + return None + + @task(task_id="get_all_vm_status_and_send_to_prometheus") + def get_all_vm_status_and_send_to_prometheus(vim_id: str): + """Authenticate against VIM, collect servers status and send to prometheus""" + + # Get VIM account info from MongoDB + print(f"Reading VIM info, id: {vim_id}") + cfg = Config() + common_db = CommonDbClient(cfg) + vim_account = common_db.get_vim_account(vim_account_id=vim_id) + print(vim_account) + + # Define Prometheus Metric for NS topology + registry = CollectorRegistry() + metric = Gauge( + PROMETHEUS_METRIC, + PROMETHEUS_METRIC_DESCRIPTION, + labelnames=[ + "vm_id", + "vim_id", + ], + registry=registry, + ) + + # Get status of all VM from VIM + all_vm_status = get_all_vm_status(vim_account) + print(f"Got {len(all_vm_status)} VMs with their status:") + if all_vm_status: + for vm in all_vm_status: + vm_id = vm["id"] + vm_status = vm["status"] + vm_name = vm.get("name", "") + print(f" {vm_name} ({vm_id}) {vm_status}") + metric.labels(vm_id, vim_id).set(vm_status) + # Push to Prometheus only if there are VM + push_to_gateway( + gateway=PROMETHEUS_PUSHGW, + job=f"{PROMETHEUS_JOB_PREFIX}{vim_id}", + registry=registry, + ) + return + + get_all_vm_status_and_send_to_prometheus(vim_id) + + return dag + + +vim_list = get_all_vim() +for index, vim in enumerate(vim_list): + vim_type = vim["vim_type"] + if vim_type in SUPPORTED_VIM_TYPES: + vim_id = vim["_id"] + vim_name = vim["name"] + dag_description = f"Dag for vim {vim_name}" + dag_id = f"vm_status_vim_{vim_id}" + print(f"Creating DAG {dag_id}") + globals()[dag_id] = create_dag( + dag_id=dag_id, + dag_number=index, + dag_description=dag_description, + vim_id=vim_id, + ) + else: + print(f"VIM type '{vim_type}' not supported for collecting VM status") diff --git a/src/dags/ns_topology.py b/src/dags/ns_topology.py new file mode 100644 index 0000000..d3fb504 --- /dev/null +++ b/src/dags/ns_topology.py @@ -0,0 +1,156 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +from datetime import datetime, timedelta + +from airflow.decorators import dag, task +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway + + +PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091" +PROMETHEUS_JOB = "airflow_osm_ns_topology" +PROMETHEUS_METRIC = "ns_topology" +PROMETHEUS_METRIC_DESCRIPTION = "Network services topology" +SCHEDULE_INTERVAL = 2 + + +@dag( + catchup=False, + default_args={ + "depends_on_past": False, + "retries": 1, + "retry_delay": timedelta(seconds=10), + }, + description="NS topology", + is_paused_upon_creation=False, + # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL), + schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *", + start_date=datetime(2022, 1, 1), + tags=["osm", "topology"], +) +def ns_topology(): + @task(task_id="get_topology") + def get_topology(): + """ + Get NS topology from MongoDB and exports it as a metric + to Prometheus + """ + + # Define Prometheus Metric for NS topology + registry = CollectorRegistry() + metric = Gauge( + PROMETHEUS_METRIC, + PROMETHEUS_METRIC_DESCRIPTION, + labelnames=[ + "ns_id", + "project_id", + "vnf_id", + "vdu_id", + "vm_id", + "vim_id", + "vdu_name", + "vnf_member_index", + ], + registry=registry, + ) + + # Getting VNFR list from MongoDB + print("Getting VNFR list from MongoDB") + cfg = Config() + print(cfg.conf) + common_db = CommonDbClient(cfg) + vnfr_list = common_db.get_vnfrs() + + # Only send topology if ns state is one of the nsAllowedStatesSet + nsAllowedStatesSet = {"INSTANTIATED"} + + # For loop to get NS topology. + # For each VDU, a metric sample is created with the appropriate labels + for vnfr in vnfr_list: + # Label ns_id + vnf_id = vnfr["_id"] + # Label ns_id + ns_id = vnfr["nsr-id-ref"] + # Label vnfd_id + vnfd_id = vnfr["vnfd-ref"] + # Label project_id + project_list = vnfr.get("_admin", {}).get("projects_read", []) + project_id = "None" + if project_list: + project_id = project_list[0] + # TODO: use logger with loglevels instead of print + # Other info + ns_state = vnfr["_admin"]["nsState"] + vnf_membex_index = vnfr["member-vnf-index-ref"] + print( + f"Read VNFR: id: {vnf_id}, ns_id: {ns_id}, ", + f"state: {ns_state}, vnfd_id: {vnfd_id}, ", + f"vnf_membex_index: {vnf_membex_index}, ", + f"project_id: {project_id}", + ) + # Only send topology if ns State is one of the nsAllowedStatesSet + if ns_state not in nsAllowedStatesSet: + continue + + print("VDU list:") + for vdu in vnfr.get("vdur", []): + # Label vdu_id + vdu_id = vdu["_id"] + # Label vim_id + vim_info = vdu.get("vim_info") + if not vim_info: + print("Error: vim_info not available in vdur") + continue + if len(vim_info) != 1: + print("Error: more than one vim_info in vdur") + continue + vim_id = next(iter(vim_info))[4:] + # Label vm_id + vm_id = vdu["vim-id"] + # Other VDU info + vdu_name = vdu.get("name", "UNKNOWN") + print( + f" id: {vdu_id}, name: {vdu_name}, " + f"vim_id: {vim_id}, vm_id: {vm_id}" + ) + print( + f"METRIC SAMPLE: ns_id: {ns_id}, ", + f"project_id: {project_id}, vnf_id: {vnf_id}, ", + f"vdu_id: {vdu_id}, vm_id: {vm_id}, vim_id: {vim_id}", + ) + metric.labels( + ns_id, + project_id, + vnf_id, + vdu_id, + vm_id, + vim_id, + vdu_name, + vnf_membex_index, + ).set(1) + + # print("Push to gateway") + push_to_gateway( + gateway=PROMETHEUS_PUSHGW, job=PROMETHEUS_JOB, registry=registry + ) + return + + get_topology() + + +dag = ns_topology() diff --git a/src/osm_mon/__init__.py b/src/osm_mon/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/osm_mon/core/__init__.py b/src/osm_mon/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/osm_mon/core/common_db.py b/src/osm_mon/core/common_db.py new file mode 100644 index 0000000..7c579c3 --- /dev/null +++ b/src/osm_mon/core/common_db.py @@ -0,0 +1,90 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +from osm_common import dbmemory, dbmongo +from osm_mon.core.config import Config + + +class CommonDbClient: + def __init__(self, config: Config): + if config.get("database", "driver") == "mongo": + self.common_db = dbmongo.DbMongo() + elif config.get("database", "driver") == "memory": + self.common_db = dbmemory.DbMemory() + else: + raise Exception( + "Unknown database driver {}".format(config.get("section", "driver")) + ) + self.common_db.db_connect(config.get("database")) + + def get_vnfr(self, nsr_id: str, member_index: int): + vnfr = self.common_db.get_one( + "vnfrs", {"nsr-id-ref": nsr_id, "member-vnf-index-ref": str(member_index)} + ) + return vnfr + + def get_vnfrs(self, nsr_id: str = None, vim_account_id: str = None): + if nsr_id and vim_account_id: + raise NotImplementedError("Only one filter is currently supported") + if nsr_id: + vnfrs = [ + self.get_vnfr(nsr_id, member["member-vnf-index"]) + for member in self.get_nsr(nsr_id)["nsd"]["constituent-vnfd"] + ] + elif vim_account_id: + vnfrs = self.common_db.get_list("vnfrs", {"vim-account-id": vim_account_id}) + else: + vnfrs = self.common_db.get_list("vnfrs") + return vnfrs + + def get_nsr(self, nsr_id: str): + nsr = self.common_db.get_one("nsrs", {"id": nsr_id}) + return nsr + + def decrypt_vim_password(self, vim_password: str, schema_version: str, vim_id: str): + return self.common_db.decrypt(vim_password, schema_version, vim_id) + + def get_vim_accounts(self): + return self.common_db.get_list("vim_accounts") + + def get_vim_account(self, vim_account_id: str) -> dict: + vim_account = self.common_db.get_one("vim_accounts", {"_id": vim_account_id}) + vim_account["vim_password"] = self.decrypt_vim_password( + vim_account["vim_password"], vim_account["schema_version"], vim_account_id + ) + vim_config_encrypted_dict = { + "1.1": ("admin_password", "nsx_password", "vcenter_password"), + "default": ( + "admin_password", + "nsx_password", + "vcenter_password", + "vrops_password", + ), + } + vim_config_encrypted = vim_config_encrypted_dict["default"] + if vim_account["schema_version"] in vim_config_encrypted_dict.keys(): + vim_config_encrypted = vim_config_encrypted_dict[ + vim_account["schema_version"] + ] + if "config" in vim_account: + for key in vim_account["config"]: + if key in vim_config_encrypted: + vim_account["config"][key] = self.decrypt_vim_password( + vim_account["config"][key], + vim_account["schema_version"], + vim_account_id, + ) + return vim_account diff --git a/src/osm_mon/core/config.py b/src/osm_mon/core/config.py new file mode 100644 index 0000000..eb0e493 --- /dev/null +++ b/src/osm_mon/core/config.py @@ -0,0 +1,68 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +"""Global configuration managed by environment variables.""" + +import logging +import os + +import pkg_resources +import yaml + +logger = logging.getLogger(__name__) + + +class Config: + def __init__(self, config_file: str = ""): + self.conf = {} + self._read_config_file(config_file) + self._read_env() + + def _read_config_file(self, config_file): + if not config_file: + path = "config.yaml" + config_file = pkg_resources.resource_filename(__name__, path) + with open(config_file) as f: + self.conf = yaml.load(f) + + def get(self, section, field=None): + if not field: + return self.conf[section] + return self.conf[section].get(field) + + def set(self, section, field, value): + if section not in self.conf: + self.conf[section] = {} + self.conf[section][field] = value + + def _read_env(self): + for env in os.environ: + if not env.startswith("OSMMON_"): + continue + elements = env.lower().split("_") + if len(elements) < 3: + logger.warning( + "Environment variable %s=%s does not comply with required format. Section and/or field missing.", + env, + os.getenv(env), + ) + continue + section = elements[1] + field = "_".join(elements[2:]) + value = os.getenv(env) + if section not in self.conf: + self.conf[section] = {} + self.conf[section][field] = value diff --git a/src/osm_mon/core/config.yaml b/src/osm_mon/core/config.yaml new file mode 100644 index 0000000..197c818 --- /dev/null +++ b/src/osm_mon/core/config.yaml @@ -0,0 +1,27 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +global: + loglevel: INFO + request_timeout: 10 + +database: + driver: mongo + uri: mongodb://mongodb-k8s:27017/?replicaSet=rs0 + name: osm + commonkey: gj7LmbCexbmII7memwbGRRdfbYuT3nvy + diff --git a/src/osm_mon/vim_connectors/__init__.py b/src/osm_mon/vim_connectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/osm_mon/vim_connectors/azure.py b/src/osm_mon/vim_connectors/azure.py new file mode 100644 index 0000000..a401f75 --- /dev/null +++ b/src/osm_mon/vim_connectors/azure.py @@ -0,0 +1,183 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +import logging +from typing import Dict, List + +from azure.identity import ClientSecretCredential +from azure.mgmt.compute import ComputeManagementClient +from azure.profiles import ProfileDefinition +from osm_mon.vim_connectors.base_vim import VIMConnector + + +log = logging.getLogger(__name__) + + +class AzureCollector(VIMConnector): + + # Translate azure provisioning state to OSM provision state. + # The first three ones are the transitional status once a user initiated + # action has been requested. Once the operation is complete, it will + # transition into the states Succeeded or Failed + # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/states-lifecycle + provision_state2osm = { + "Creating": "BUILD", + "Updating": "BUILD", + "Deleting": "INACTIVE", + "Succeeded": "ACTIVE", + "Failed": "ERROR", + } + + # Translate azure power state to OSM provision state + power_state2osm = { + "starting": "INACTIVE", + "running": "ACTIVE", + "stopping": "INACTIVE", + "stopped": "INACTIVE", + "unknown": "OTHER", + "deallocated": "BUILD", + "deallocating": "BUILD", + } + + AZURE_COMPUTE_MGMT_CLIENT_API_VERSION = "2021-03-01" + AZURE_COMPUTE_MGMT_PROFILE_TAG = "azure.mgmt.compute.ComputeManagementClient" + AZURE_COMPUTE_MGMT_PROFILE = ProfileDefinition( + { + AZURE_COMPUTE_MGMT_PROFILE_TAG: { + None: AZURE_COMPUTE_MGMT_CLIENT_API_VERSION, + "availability_sets": "2020-12-01", + "dedicated_host_groups": "2020-12-01", + "dedicated_hosts": "2020-12-01", + "disk_accesses": "2020-12-01", + "disk_encryption_sets": "2020-12-01", + "disk_restore_point": "2020-12-01", + "disks": "2020-12-01", + "galleries": "2020-09-30", + "gallery_application_versions": "2020-09-30", + "gallery_applications": "2020-09-30", + "gallery_image_versions": "2020-09-30", + "gallery_images": "2020-09-30", + "gallery_sharing_profile": "2020-09-30", + "images": "2020-12-01", + "log_analytics": "2020-12-01", + "operations": "2020-12-01", + "proximity_placement_groups": "2020-12-01", + "resource_skus": "2019-04-01", + "shared_galleries": "2020-09-30", + "shared_gallery_image_versions": "2020-09-30", + "shared_gallery_images": "2020-09-30", + "snapshots": "2020-12-01", + "ssh_public_keys": "2020-12-01", + "usage": "2020-12-01", + "virtual_machine_extension_images": "2020-12-01", + "virtual_machine_extensions": "2020-12-01", + "virtual_machine_images": "2020-12-01", + "virtual_machine_images_edge_zone": "2020-12-01", + "virtual_machine_run_commands": "2020-12-01", + "virtual_machine_scale_set_extensions": "2020-12-01", + "virtual_machine_scale_set_rolling_upgrades": "2020-12-01", + "virtual_machine_scale_set_vm_extensions": "2020-12-01", + "virtual_machine_scale_set_vm_run_commands": "2020-12-01", + "virtual_machine_scale_set_vms": "2020-12-01", + "virtual_machine_scale_sets": "2020-12-01", + "virtual_machine_sizes": "2020-12-01", + "virtual_machines": "2020-12-01", + } + }, + AZURE_COMPUTE_MGMT_PROFILE_TAG + " osm", + ) + + def __init__(self, vim_account: Dict): + self.vim_account = vim_account + self.reload_client = True + logger = logging.getLogger("azure") + logger.setLevel(logging.ERROR) + # Store config to create azure subscription later + self._config = { + "user": vim_account["vim_user"], + "passwd": vim_account["vim_password"], + "tenant": vim_account["vim_tenant_name"], + } + + # SUBSCRIPTION + config = vim_account["config"] + if "subscription_id" in config: + self._config["subscription_id"] = config.get("subscription_id") + log.info("Subscription: %s", self._config["subscription_id"]) + else: + log.error("Subscription not specified") + return + + # RESOURCE_GROUP + if "resource_group" in config: + self.resource_group = config.get("resource_group") + else: + log.error("Azure resource_group is not specified at config") + return + + def _reload_connection(self): + if self.reload_client: + log.debug("reloading azure client") + try: + self.credentials = ClientSecretCredential( + client_id=self._config["user"], + client_secret=self._config["passwd"], + tenant_id=self._config["tenant"], + ) + self.conn_compute = ComputeManagementClient( + self.credentials, + self._config["subscription_id"], + profile=self.AZURE_COMPUTE_MGMT_PROFILE, + ) + # Set to client created + self.reload_client = False + except Exception as e: + log.error(e) + + def collect_servers_status(self) -> List[Dict]: + servers = [] + log.debug("collect_servers_status") + self._reload_connection() + try: + for vm in self.conn_compute.virtual_machines.list(self.resource_group): + id = vm.id + array = id.split("/") + name = array[-1] + status = self.provision_state2osm.get(vm.provisioning_state, "OTHER") + if vm.provisioning_state == "Succeeded": + # check if machine is running or stopped + instance_view = self.conn_compute.virtual_machines.instance_view( + self.resource_group, name + ) + for status in instance_view.statuses: + splitted_status = status.code.split("/") + if ( + len(splitted_status) == 2 + and splitted_status[0] == "PowerState" + ): + status = self.power_state2osm.get( + splitted_status[1], "OTHER" + ) + # log.info(f'id: {id}, name: {name}, status: {status}') + vm = { + "id": id, + "name": name, + "status": (1 if (status == "ACTIVE") else 0), + } + servers.append(vm) + except Exception as e: + log.error(e) + return servers diff --git a/src/osm_mon/vim_connectors/base_vim.py b/src/osm_mon/vim_connectors/base_vim.py new file mode 100644 index 0000000..9ec2cde --- /dev/null +++ b/src/osm_mon/vim_connectors/base_vim.py @@ -0,0 +1,26 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +from typing import Dict, List + + +class VIMConnector: + def __init__(self, vim_account: Dict): + pass + + # def collect_servers_status(self) -> List[Metric]: + def collect_servers_status(self) -> List: + pass diff --git a/src/osm_mon/vim_connectors/gcp.py b/src/osm_mon/vim_connectors/gcp.py new file mode 100644 index 0000000..6c7b557 --- /dev/null +++ b/src/osm_mon/vim_connectors/gcp.py @@ -0,0 +1,92 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +# pylint: disable=E1101 + +import logging +from typing import Dict, List + +from google.oauth2 import service_account +import googleapiclient.discovery +from osm_mon.vim_connectors.base_vim import VIMConnector + +log = logging.getLogger(__name__) + + +class GcpCollector(VIMConnector): + def __init__(self, vim_account: Dict): + self.vim_account = vim_account + self.project = vim_account["vim_tenant_name"] or vim_account["vim_tenant_id"] + + # REGION - Google Cloud considers regions and zones. A specific region + # can have more than one zone (for instance: region us-west1 with the + # zones us-west1-a, us-west1-b and us-west1-c). So the region name + # specified in the config will be considered as a specific zone for GC + # and the region will be calculated from that without the preffix. + if "config" in vim_account: + config = vim_account["config"] + if "region_name" in config: + self.zone = config.get("region_name") + self.region = self.zone.rsplit("-", 1)[0] + else: + log.error("Google Cloud region_name not specified in config") + else: + log.error("config is not specified in VIM") + + # Credentials + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + self.credentials = None + if "credentials" in config: + log.debug("Setting credentials") + # Settings Google Cloud credentials dict + creds_body = config["credentials"] + creds = service_account.Credentials.from_service_account_info(creds_body) + if "sa_file" in config: + creds = service_account.Credentials.from_service_account_file( + config.get("sa_file"), scopes=scopes + ) + log.debug("Credentials: %s", creds) + # Construct a Resource for interacting with an API. + self.credentials = creds + try: + self.conn_compute = googleapiclient.discovery.build( + "compute", "v1", credentials=creds + ) + except Exception as e: + log.error(e) + else: + log.error("It is not possible to init GCP with no credentials") + + def collect_servers_status(self) -> List[Dict]: + servers = [] + try: + response = ( + self.conn_compute.instances() + .list(project=self.project, zone=self.zone) + .execute() + ) + if "items" in response: + log.info(response["items"]) + for server in response["items"]: + vm = { + "id": server["id"], + "name": server["name"], + "status": (1 if (server["status"] == "RUNNING") else 0), + } + servers.append(vm) + except Exception as e: + log.error(e) + return servers diff --git a/src/osm_mon/vim_connectors/openstack.py b/src/osm_mon/vim_connectors/openstack.py new file mode 100644 index 0000000..67ca4c5 --- /dev/null +++ b/src/osm_mon/vim_connectors/openstack.py @@ -0,0 +1,83 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +import logging +from typing import Dict, List + +from keystoneauth1 import session +from keystoneauth1.identity import v3 +from novaclient import client as nova_client +from osm_mon.vim_connectors.base_vim import VIMConnector + +log = logging.getLogger(__name__) + + +class CertificateNotCreated(Exception): + pass + + +class OpenStackCollector(VIMConnector): + def __init__(self, vim_account: Dict): + log.info("__init__") + self.vim_account = vim_account + self.vim_session = None + self.vim_session = self._get_session(vim_account) + self.nova = self._build_nova_client() + + def _get_session(self, creds: Dict): + verify_ssl = True + project_domain_name = "Default" + user_domain_name = "Default" + try: + if "config" in creds: + vim_config = creds["config"] + if "insecure" in vim_config and vim_config["insecure"]: + verify_ssl = False + if "ca_cert" in vim_config: + verify_ssl = vim_config["ca_cert"] + elif "ca_cert_content" in vim_config: + # vim_config = self._create_file_cert(vim_config, creds["_id"]) + verify_ssl = vim_config["ca_cert"] + if "project_domain_name" in vim_config: + project_domain_name = vim_config["project_domain_name"] + if "user_domain_name" in vim_config: + user_domain_name = vim_config["user_domain_name"] + auth = v3.Password( + auth_url=creds["vim_url"], + username=creds["vim_user"], + password=creds["vim_password"], + project_name=creds["vim_tenant_name"], + project_domain_name=project_domain_name, + user_domain_name=user_domain_name, + ) + return session.Session(auth=auth, verify=verify_ssl, timeout=10) + except CertificateNotCreated as e: + log.error(e) + + def _build_nova_client(self) -> nova_client.Client: + return nova_client.Client("2", session=self.vim_session, timeout=10) + + def collect_servers_status(self) -> List[Dict]: + log.info("collect_servers_status") + servers = [] + for server in self.nova.servers.list(detailed=True): + vm = { + "id": server.id, + "name": server.name, + "status": (0 if (server.status == "ERROR") else 1), + } + servers.append(vm) + return servers diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..b29a414 --- /dev/null +++ b/tox.ini @@ -0,0 +1,126 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +[tox] +envlist = black, flake8 + +[tox:jenkins] +toxworkdir = /tmp/.tox +setenv = XDG_CACHE_HOME=/tmp/.cache + +[testenv] +usedevelop = True +basepython = python3 +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE = 1 +deps = -r{toxinidir}/requirements.txt +parallel_show_output = true + + +####################################################################################### +[testenv:black] +deps = black +skip_install = true +commands = + black --check --diff src + + +####################################################################################### +[testenv:cover] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt +whitelist_externals = sh +commands = + sh -c 'rm -f nosetests.xml' + coverage erase + nose2 -C --coverage src -s src + sh -c 'mv .coverage .coverage_mon' + coverage report --omit='*tests*' + coverage html -d ./cover --omit='*tests*' + coverage xml -o coverage.xml --omit='*tests*' + + +####################################################################################### +[testenv:flake8] +deps = flake8 + flake8-import-order +skip_install = true +commands = + flake8 src/ + + +####################################################################################### +[testenv:pylint] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt + pylint +skip_install = true +commands = + pylint -E src + + +####################################################################################### +[testenv:safety] +setenv = + LC_ALL=C.UTF-8 + LANG=C.UTF-8 +deps = {[testenv]deps} + safety +commands = + - safety check --full-report + + +####################################################################################### +[testenv:pip-compile] +deps = pip-tools==6.6.2 +skip_install = true +whitelist_externals = + bash + [ +commands = + bash -c "for file in requirements*.in ; do \ + UNSAFE="" ; \ + if [[ $file =~ 'dist' ]] ; then UNSAFE='--allow-unsafe' ; fi ; \ + pip-compile -rU --no-header $UNSAFE $file ;\ + out=`echo $file | sed 's/.in/.txt/'` ; \ + sed -i -e '1 e head -16 tox.ini' $out ;\ + done" + + +####################################################################################### +[flake8] +ignore = + W291, + W293, + W503, + W605, + E123, + E125, + E203, + E226, + E241, + E501, +exclude = + .git, + __pycache__, + .tox, +max-line-length = 120 +show-source = True +builtins = _ +import-order-style = google -- 2.25.1