From: garciadeblas Date: Thu, 24 Nov 2022 13:42:35 +0000 (+0100) Subject: Deb package creation for NG-SA X-Git-Tag: v12.0.3 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=260b704249d550e85438aff2eb64fbd9ab889eb8;p=osm%2FNG-SA.git Deb package creation for NG-SA Change-Id: I96bb72bfd77e0c57215ba62abdbdaf8df5bb8cde Signed-off-by: garciadeblas --- diff --git a/.gitignore b/.gitignore index 4f7530f..fb6947e 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ dist *.local local +src/osm_ngsa/_version.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..fc5b284 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,20 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +include src/osm_ngsa/requirements.txt +include src/osm_ngsa/README.rst + diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..817570a --- /dev/null +++ b/README.rst @@ -0,0 +1,24 @@ +.. + ####################################################################################### + # Copyright ETSI Contributors and Others. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + # implied. + # See the License for the specific language governing permissions and + # limitations under the License. + ####################################################################################### + +=========== +osm-ngsa +=========== + +osm-ngsa is the new Service Assurance Python library for OSM. It includes the different Airflow DAGs used for specific monitoring and alerting stages, as well as common libraries used by those DAGs to interact with VIM. + diff --git a/devops-stages/stage-archive.sh b/devops-stages/stage-archive.sh index 7c88360..e70b172 100755 --- a/devops-stages/stage-archive.sh +++ b/devops-stages/stage-archive.sh @@ -16,4 +16,13 @@ # limitations under the License. # -echo "Nothing to be done" +MDG=NG-SA +rm -rf pool +rm -rf dists +mkdir -p pool/$MDG +mv deb_dist/*.deb pool/$MDG/ +mkdir -p dists/unstable/$MDG/binary-amd64/ +apt-ftparchive packages pool/$MDG > dists/unstable/$MDG/binary-amd64/Packages +gzip -9fk dists/unstable/$MDG/binary-amd64/Packages +echo "dists/**,pool/$MDG/*.deb" + diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index 7c88360..7b483a8 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -16,4 +16,6 @@ # limitations under the License. # -echo "Nothing to be done" +rm -rf dist deb_dist osm*.tar.gz *.egg-info .eggs + +tox -e dist diff --git a/requirements-dev.txt b/requirements-dev.txt index c848d9e..3d50187 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -175,7 +175,7 @@ jinja2==3.1.2 # flask-babel # python-nvd3 # swagger-ui-bundle -jsonschema==4.17.0 +jsonschema==4.17.1 # via # apache-airflow # connexion diff --git a/requirements-dist.in b/requirements-dist.in new file mode 100644 index 0000000..03ff6e9 --- /dev/null +++ b/requirements-dist.in @@ -0,0 +1,18 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stdeb +setuptools-scm +setuptools<60 diff --git a/requirements-dist.txt b/requirements-dist.txt new file mode 100644 index 0000000..52c8aa0 --- /dev/null +++ b/requirements-dist.txt @@ -0,0 +1,34 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +packaging==21.3 + # via setuptools-scm +pyparsing==3.0.9 + # via packaging +setuptools-scm==7.0.5 + # via -r requirements-dist.in +stdeb==0.10.0 + # via -r requirements-dist.in +tomli==2.0.1 + # via setuptools-scm +typing-extensions==4.4.0 + # via setuptools-scm + +# The following packages are considered to be unsafe in a requirements file: +setuptools==59.8.0 + # via + # -r requirements-dist.in + # setuptools-scm diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..844bb93 --- /dev/null +++ b/setup.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from setuptools import setup, find_packages, find_namespace_packages + +_name = "osm_ngsa" +_description = "OSM Service Assurance Airflow DAGs and libraries" +with open(os.path.join(".", "README.rst")) as readme_file: + README = readme_file.read() + +setup( + name=_name, + description=_description, + long_description=README, + use_scm_version={ + "write_to": "src/osm_ngsa/_version.py" + }, + author="ETSI OSM", + author_email="osmsupport@etsi.org", + maintainer="ETSI OSM", + maintainer_email="osmsupport@etsi.org", + url="https://osm.etsi.org/gitweb/?p=osm/NG-SA.git;a=summary", + license="Apache 2.0", + package_dir={"": "src"}, + packages=find_namespace_packages(where='src'), + include_package_data=True, + setup_requires=["setuptools-scm"], +) + diff --git a/src/dags/.airflowignore b/src/dags/.airflowignore deleted file mode 100644 index 8b13789..0000000 --- a/src/dags/.airflowignore +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/dags/__init__.py b/src/dags/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/dags/multivim_vim_status.py b/src/dags/multivim_vim_status.py deleted file mode 100644 index 93894b1..0000000 --- a/src/dags/multivim_vim_status.py +++ /dev/null @@ -1,153 +0,0 @@ -####################################################################################### -# Copyright ETSI Contributors and Others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################################### -from datetime import datetime, timedelta - -from airflow import DAG -from airflow.decorators import task -from osm_mon.core.common_db import CommonDbClient -from osm_mon.core.config import Config -from osm_mon.vim_connectors.azure import AzureCollector -from osm_mon.vim_connectors.gcp import GcpCollector -from osm_mon.vim_connectors.openstack import OpenStackCollector -from prometheus_client import CollectorRegistry, Gauge, push_to_gateway - - -SUPPORTED_VIM_TYPES = ["openstack", "vio", "gcp", "azure"] -PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091" -PROMETHEUS_JOB_PREFIX = "airflow_osm_vim_status_" -PROMETHEUS_METRIC = "vim_status" -PROMETHEUS_METRIC_DESCRIPTION = "VIM status" -SCHEDULE_INTERVAL = 1 - - -def get_all_vim(): - """Get VIMs from MongoDB""" - print("Getting VIM list") - - cfg = Config() - print(cfg.conf) - common_db = CommonDbClient(cfg) - vim_accounts = common_db.get_vim_accounts() - vim_list = [] - for vim in vim_accounts: - print(f'Read VIM {vim["_id"]} ({vim["name"]})') - vim_list.append( - {"_id": vim["_id"], "name": vim["name"], "vim_type": vim["vim_type"]} - ) - - print(vim_list) - print("Getting VIM list OK") - return vim_list - - -def create_dag(dag_id, dag_number, dag_description, vim_id): - dag = DAG( - dag_id, - catchup=False, - default_args={ - "depends_on_past": False, - "retries": 1, - # "retry_delay": timedelta(minutes=1), - "retry_delay": timedelta(seconds=10), - }, - description=dag_description, - is_paused_upon_creation=False, - # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL), - schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *", - start_date=datetime(2022, 1, 1), - tags=["osm", "vim"], - ) - - with dag: - - def get_vim_collector(vim_account): - """Return a VIM collector for the vim_account""" - vim_type = vim_account["vim_type"] - if "config" in vim_account and "vim_type" in vim_account["config"]: - vim_type = vim_account["config"]["vim_type"].lower() - if vim_type == "vio" and "vrops_site" not in vim_account["config"]: - vim_type = "openstack" - if vim_type == "openstack": - return OpenStackCollector(vim_account) - if vim_type == "gcp": - return GcpCollector(vim_account) - if vim_type == "azure": - return AzureCollector(vim_account) - print(f"VIM type '{vim_type}' not supported") - return None - - @task(task_id="get_vim_status_and_send_to_prometheus") - def get_vim_status_and_send_to_prometheus(vim_id: str): - """Authenticate against VIM and check status""" - - # Get VIM account info from MongoDB - print(f"Reading VIM info, id: {vim_id}") - cfg = Config() - common_db = CommonDbClient(cfg) - vim_account = common_db.get_vim_account(vim_account_id=vim_id) - print(vim_account) - - # Define Prometheus Metric for NS topology - registry = CollectorRegistry() - metric = Gauge( - PROMETHEUS_METRIC, - PROMETHEUS_METRIC_DESCRIPTION, - labelnames=[ - "vim_id", - ], - registry=registry, - ) - metric.labels(vim_id).set(0) - - # Get status of VIM - collector = get_vim_collector(vim_account) - if collector: - status = collector.is_vim_ok() - print(f"VIM status: {status}") - metric.labels(vim_id).set(1) - else: - print("Error creating VIM collector") - # Push to Prometheus - push_to_gateway( - gateway=PROMETHEUS_PUSHGW, - job=f"{PROMETHEUS_JOB_PREFIX}{vim_id}", - registry=registry, - ) - return - - get_vim_status_and_send_to_prometheus(vim_id) - - return dag - - -vim_list = get_all_vim() -for index, vim in enumerate(vim_list): - vim_type = vim["vim_type"] - if vim_type in SUPPORTED_VIM_TYPES: - vim_id = vim["_id"] - vim_name = vim["name"] - dag_description = f"Dag for VIM {vim_name} status" - dag_id = f"vim_status_{vim_id}" - print(f"Creating DAG {dag_id}") - globals()[dag_id] = create_dag( - dag_id=dag_id, - dag_number=index, - dag_description=dag_description, - vim_id=vim_id, - ) - else: - print(f"VIM type '{vim_type}' not supported for monitoring VIM status") diff --git a/src/dags/multivim_vm_status.py b/src/dags/multivim_vm_status.py deleted file mode 100644 index dbdbbc0..0000000 --- a/src/dags/multivim_vm_status.py +++ /dev/null @@ -1,166 +0,0 @@ -####################################################################################### -# Copyright ETSI Contributors and Others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################################### -from datetime import datetime, timedelta - -from airflow import DAG -from airflow.decorators import task -from osm_mon.core.common_db import CommonDbClient -from osm_mon.core.config import Config -from osm_mon.vim_connectors.azure import AzureCollector -from osm_mon.vim_connectors.gcp import GcpCollector -from osm_mon.vim_connectors.openstack import OpenStackCollector -from prometheus_client import CollectorRegistry, Gauge, push_to_gateway - - -SUPPORTED_VIM_TYPES = ["openstack", "vio", "gcp", "azure"] -PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091" -PROMETHEUS_JOB_PREFIX = "airflow_osm_vm_status_" -PROMETHEUS_METRIC = "vm_status" -PROMETHEUS_METRIC_DESCRIPTION = "VM Status from VIM" -SCHEDULE_INTERVAL = 1 - - -def get_all_vim(): - """Get VIMs from MongoDB""" - print("Getting VIM list") - - cfg = Config() - print(cfg.conf) - common_db = CommonDbClient(cfg) - vim_accounts = common_db.get_vim_accounts() - vim_list = [] - for vim in vim_accounts: - print(f'Read VIM {vim["_id"]} ({vim["name"]})') - vim_list.append( - {"_id": vim["_id"], "name": vim["name"], "vim_type": vim["vim_type"]} - ) - - print(vim_list) - print("Getting VIM list OK") - return vim_list - - -def create_dag(dag_id, dag_number, dag_description, vim_id): - dag = DAG( - dag_id, - catchup=False, - default_args={ - "depends_on_past": False, - "retries": 1, - # "retry_delay": timedelta(minutes=1), - "retry_delay": timedelta(seconds=10), - }, - description=dag_description, - is_paused_upon_creation=False, - # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL), - schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *", - start_date=datetime(2022, 1, 1), - tags=["osm", "vim"], - ) - - with dag: - - def get_vim_collector(vim_account): - """Return a VIM collector for the vim_account""" - vim_type = vim_account["vim_type"] - if "config" in vim_account and "vim_type" in vim_account["config"]: - vim_type = vim_account["config"]["vim_type"].lower() - if vim_type == "vio" and "vrops_site" not in vim_account["config"]: - vim_type = "openstack" - if vim_type == "openstack": - return OpenStackCollector(vim_account) - if vim_type == "gcp": - return GcpCollector(vim_account) - if vim_type == "azure": - return AzureCollector(vim_account) - print(f"VIM type '{vim_type}' not supported") - return None - - def get_all_vm_status(vim_account): - """Get VM status from the VIM""" - collector = get_vim_collector(vim_account) - if collector: - status = collector.is_vim_ok() - print(f"VIM status: {status}") - vm_status_list = collector.collect_servers_status() - return vm_status_list - else: - return None - - @task(task_id="get_all_vm_status_and_send_to_prometheus") - def get_all_vm_status_and_send_to_prometheus(vim_id: str): - """Authenticate against VIM, collect servers status and send to prometheus""" - - # Get VIM account info from MongoDB - print(f"Reading VIM info, id: {vim_id}") - cfg = Config() - common_db = CommonDbClient(cfg) - vim_account = common_db.get_vim_account(vim_account_id=vim_id) - print(vim_account) - - # Define Prometheus Metric for NS topology - registry = CollectorRegistry() - metric = Gauge( - PROMETHEUS_METRIC, - PROMETHEUS_METRIC_DESCRIPTION, - labelnames=[ - "vm_id", - "vim_id", - ], - registry=registry, - ) - - # Get status of all VM from VIM - all_vm_status = get_all_vm_status(vim_account) - print(f"Got {len(all_vm_status)} VMs with their status:") - if all_vm_status: - for vm in all_vm_status: - vm_id = vm["id"] - vm_status = vm["status"] - vm_name = vm.get("name", "") - print(f" {vm_name} ({vm_id}) {vm_status}") - metric.labels(vm_id, vim_id).set(vm_status) - # Push to Prometheus only if there are VM - push_to_gateway( - gateway=PROMETHEUS_PUSHGW, - job=f"{PROMETHEUS_JOB_PREFIX}{vim_id}", - registry=registry, - ) - return - - get_all_vm_status_and_send_to_prometheus(vim_id) - - return dag - - -vim_list = get_all_vim() -for index, vim in enumerate(vim_list): - vim_type = vim["vim_type"] - if vim_type in SUPPORTED_VIM_TYPES: - vim_id = vim["_id"] - vim_name = vim["name"] - dag_description = f"Dag for vim {vim_name}" - dag_id = f"vm_status_vim_{vim_id}" - print(f"Creating DAG {dag_id}") - globals()[dag_id] = create_dag( - dag_id=dag_id, - dag_number=index, - dag_description=dag_description, - vim_id=vim_id, - ) - else: - print(f"VIM type '{vim_type}' not supported for collecting VM status") diff --git a/src/dags/ns_topology.py b/src/dags/ns_topology.py deleted file mode 100644 index d3fb504..0000000 --- a/src/dags/ns_topology.py +++ /dev/null @@ -1,156 +0,0 @@ -####################################################################################### -# Copyright ETSI Contributors and Others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################################### -from datetime import datetime, timedelta - -from airflow.decorators import dag, task -from osm_mon.core.common_db import CommonDbClient -from osm_mon.core.config import Config -from prometheus_client import CollectorRegistry, Gauge, push_to_gateway - - -PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091" -PROMETHEUS_JOB = "airflow_osm_ns_topology" -PROMETHEUS_METRIC = "ns_topology" -PROMETHEUS_METRIC_DESCRIPTION = "Network services topology" -SCHEDULE_INTERVAL = 2 - - -@dag( - catchup=False, - default_args={ - "depends_on_past": False, - "retries": 1, - "retry_delay": timedelta(seconds=10), - }, - description="NS topology", - is_paused_upon_creation=False, - # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL), - schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *", - start_date=datetime(2022, 1, 1), - tags=["osm", "topology"], -) -def ns_topology(): - @task(task_id="get_topology") - def get_topology(): - """ - Get NS topology from MongoDB and exports it as a metric - to Prometheus - """ - - # Define Prometheus Metric for NS topology - registry = CollectorRegistry() - metric = Gauge( - PROMETHEUS_METRIC, - PROMETHEUS_METRIC_DESCRIPTION, - labelnames=[ - "ns_id", - "project_id", - "vnf_id", - "vdu_id", - "vm_id", - "vim_id", - "vdu_name", - "vnf_member_index", - ], - registry=registry, - ) - - # Getting VNFR list from MongoDB - print("Getting VNFR list from MongoDB") - cfg = Config() - print(cfg.conf) - common_db = CommonDbClient(cfg) - vnfr_list = common_db.get_vnfrs() - - # Only send topology if ns state is one of the nsAllowedStatesSet - nsAllowedStatesSet = {"INSTANTIATED"} - - # For loop to get NS topology. - # For each VDU, a metric sample is created with the appropriate labels - for vnfr in vnfr_list: - # Label ns_id - vnf_id = vnfr["_id"] - # Label ns_id - ns_id = vnfr["nsr-id-ref"] - # Label vnfd_id - vnfd_id = vnfr["vnfd-ref"] - # Label project_id - project_list = vnfr.get("_admin", {}).get("projects_read", []) - project_id = "None" - if project_list: - project_id = project_list[0] - # TODO: use logger with loglevels instead of print - # Other info - ns_state = vnfr["_admin"]["nsState"] - vnf_membex_index = vnfr["member-vnf-index-ref"] - print( - f"Read VNFR: id: {vnf_id}, ns_id: {ns_id}, ", - f"state: {ns_state}, vnfd_id: {vnfd_id}, ", - f"vnf_membex_index: {vnf_membex_index}, ", - f"project_id: {project_id}", - ) - # Only send topology if ns State is one of the nsAllowedStatesSet - if ns_state not in nsAllowedStatesSet: - continue - - print("VDU list:") - for vdu in vnfr.get("vdur", []): - # Label vdu_id - vdu_id = vdu["_id"] - # Label vim_id - vim_info = vdu.get("vim_info") - if not vim_info: - print("Error: vim_info not available in vdur") - continue - if len(vim_info) != 1: - print("Error: more than one vim_info in vdur") - continue - vim_id = next(iter(vim_info))[4:] - # Label vm_id - vm_id = vdu["vim-id"] - # Other VDU info - vdu_name = vdu.get("name", "UNKNOWN") - print( - f" id: {vdu_id}, name: {vdu_name}, " - f"vim_id: {vim_id}, vm_id: {vm_id}" - ) - print( - f"METRIC SAMPLE: ns_id: {ns_id}, ", - f"project_id: {project_id}, vnf_id: {vnf_id}, ", - f"vdu_id: {vdu_id}, vm_id: {vm_id}, vim_id: {vim_id}", - ) - metric.labels( - ns_id, - project_id, - vnf_id, - vdu_id, - vm_id, - vim_id, - vdu_name, - vnf_membex_index, - ).set(1) - - # print("Push to gateway") - push_to_gateway( - gateway=PROMETHEUS_PUSHGW, job=PROMETHEUS_JOB, registry=registry - ) - return - - get_topology() - - -dag = ns_topology() diff --git a/src/osm_mon/__init__.py b/src/osm_mon/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/osm_mon/core/__init__.py b/src/osm_mon/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/osm_mon/core/common_db.py b/src/osm_mon/core/common_db.py deleted file mode 100644 index 7c579c3..0000000 --- a/src/osm_mon/core/common_db.py +++ /dev/null @@ -1,90 +0,0 @@ -####################################################################################### -# Copyright ETSI Contributors and Others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################################### -from osm_common import dbmemory, dbmongo -from osm_mon.core.config import Config - - -class CommonDbClient: - def __init__(self, config: Config): - if config.get("database", "driver") == "mongo": - self.common_db = dbmongo.DbMongo() - elif config.get("database", "driver") == "memory": - self.common_db = dbmemory.DbMemory() - else: - raise Exception( - "Unknown database driver {}".format(config.get("section", "driver")) - ) - self.common_db.db_connect(config.get("database")) - - def get_vnfr(self, nsr_id: str, member_index: int): - vnfr = self.common_db.get_one( - "vnfrs", {"nsr-id-ref": nsr_id, "member-vnf-index-ref": str(member_index)} - ) - return vnfr - - def get_vnfrs(self, nsr_id: str = None, vim_account_id: str = None): - if nsr_id and vim_account_id: - raise NotImplementedError("Only one filter is currently supported") - if nsr_id: - vnfrs = [ - self.get_vnfr(nsr_id, member["member-vnf-index"]) - for member in self.get_nsr(nsr_id)["nsd"]["constituent-vnfd"] - ] - elif vim_account_id: - vnfrs = self.common_db.get_list("vnfrs", {"vim-account-id": vim_account_id}) - else: - vnfrs = self.common_db.get_list("vnfrs") - return vnfrs - - def get_nsr(self, nsr_id: str): - nsr = self.common_db.get_one("nsrs", {"id": nsr_id}) - return nsr - - def decrypt_vim_password(self, vim_password: str, schema_version: str, vim_id: str): - return self.common_db.decrypt(vim_password, schema_version, vim_id) - - def get_vim_accounts(self): - return self.common_db.get_list("vim_accounts") - - def get_vim_account(self, vim_account_id: str) -> dict: - vim_account = self.common_db.get_one("vim_accounts", {"_id": vim_account_id}) - vim_account["vim_password"] = self.decrypt_vim_password( - vim_account["vim_password"], vim_account["schema_version"], vim_account_id - ) - vim_config_encrypted_dict = { - "1.1": ("admin_password", "nsx_password", "vcenter_password"), - "default": ( - "admin_password", - "nsx_password", - "vcenter_password", - "vrops_password", - ), - } - vim_config_encrypted = vim_config_encrypted_dict["default"] - if vim_account["schema_version"] in vim_config_encrypted_dict.keys(): - vim_config_encrypted = vim_config_encrypted_dict[ - vim_account["schema_version"] - ] - if "config" in vim_account: - for key in vim_account["config"]: - if key in vim_config_encrypted: - vim_account["config"][key] = self.decrypt_vim_password( - vim_account["config"][key], - vim_account["schema_version"], - vim_account_id, - ) - return vim_account diff --git a/src/osm_mon/core/config.py b/src/osm_mon/core/config.py deleted file mode 100644 index eb0e493..0000000 --- a/src/osm_mon/core/config.py +++ /dev/null @@ -1,68 +0,0 @@ -####################################################################################### -# Copyright ETSI Contributors and Others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################################### -"""Global configuration managed by environment variables.""" - -import logging -import os - -import pkg_resources -import yaml - -logger = logging.getLogger(__name__) - - -class Config: - def __init__(self, config_file: str = ""): - self.conf = {} - self._read_config_file(config_file) - self._read_env() - - def _read_config_file(self, config_file): - if not config_file: - path = "config.yaml" - config_file = pkg_resources.resource_filename(__name__, path) - with open(config_file) as f: - self.conf = yaml.load(f) - - def get(self, section, field=None): - if not field: - return self.conf[section] - return self.conf[section].get(field) - - def set(self, section, field, value): - if section not in self.conf: - self.conf[section] = {} - self.conf[section][field] = value - - def _read_env(self): - for env in os.environ: - if not env.startswith("OSMMON_"): - continue - elements = env.lower().split("_") - if len(elements) < 3: - logger.warning( - "Environment variable %s=%s does not comply with required format. Section and/or field missing.", - env, - os.getenv(env), - ) - continue - section = elements[1] - field = "_".join(elements[2:]) - value = os.getenv(env) - if section not in self.conf: - self.conf[section] = {} - self.conf[section][field] = value diff --git a/src/osm_mon/core/config.yaml b/src/osm_mon/core/config.yaml deleted file mode 100644 index 197c818..0000000 --- a/src/osm_mon/core/config.yaml +++ /dev/null @@ -1,27 +0,0 @@ -####################################################################################### -# Copyright ETSI Contributors and Others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################################### - -global: - loglevel: INFO - request_timeout: 10 - -database: - driver: mongo - uri: mongodb://mongodb-k8s:27017/?replicaSet=rs0 - name: osm - commonkey: gj7LmbCexbmII7memwbGRRdfbYuT3nvy - diff --git a/src/osm_mon/vim_connectors/__init__.py b/src/osm_mon/vim_connectors/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/osm_mon/vim_connectors/azure.py b/src/osm_mon/vim_connectors/azure.py deleted file mode 100644 index 23086c4..0000000 --- a/src/osm_mon/vim_connectors/azure.py +++ /dev/null @@ -1,193 +0,0 @@ -####################################################################################### -# Copyright ETSI Contributors and Others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################################### -import logging -from typing import Dict, List - -from azure.identity import ClientSecretCredential -from azure.mgmt.compute import ComputeManagementClient -from azure.profiles import ProfileDefinition -from osm_mon.vim_connectors.base_vim import VIMConnector - - -log = logging.getLogger(__name__) - - -class AzureCollector(VIMConnector): - - # Translate azure provisioning state to OSM provision state. - # The first three ones are the transitional status once a user initiated - # action has been requested. Once the operation is complete, it will - # transition into the states Succeeded or Failed - # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/states-lifecycle - provision_state2osm = { - "Creating": "BUILD", - "Updating": "BUILD", - "Deleting": "INACTIVE", - "Succeeded": "ACTIVE", - "Failed": "ERROR", - } - - # Translate azure power state to OSM provision state - power_state2osm = { - "starting": "INACTIVE", - "running": "ACTIVE", - "stopping": "INACTIVE", - "stopped": "INACTIVE", - "unknown": "OTHER", - "deallocated": "BUILD", - "deallocating": "BUILD", - } - - AZURE_COMPUTE_MGMT_CLIENT_API_VERSION = "2021-03-01" - AZURE_COMPUTE_MGMT_PROFILE_TAG = "azure.mgmt.compute.ComputeManagementClient" - AZURE_COMPUTE_MGMT_PROFILE = ProfileDefinition( - { - AZURE_COMPUTE_MGMT_PROFILE_TAG: { - None: AZURE_COMPUTE_MGMT_CLIENT_API_VERSION, - "availability_sets": "2020-12-01", - "dedicated_host_groups": "2020-12-01", - "dedicated_hosts": "2020-12-01", - "disk_accesses": "2020-12-01", - "disk_encryption_sets": "2020-12-01", - "disk_restore_point": "2020-12-01", - "disks": "2020-12-01", - "galleries": "2020-09-30", - "gallery_application_versions": "2020-09-30", - "gallery_applications": "2020-09-30", - "gallery_image_versions": "2020-09-30", - "gallery_images": "2020-09-30", - "gallery_sharing_profile": "2020-09-30", - "images": "2020-12-01", - "log_analytics": "2020-12-01", - "operations": "2020-12-01", - "proximity_placement_groups": "2020-12-01", - "resource_skus": "2019-04-01", - "shared_galleries": "2020-09-30", - "shared_gallery_image_versions": "2020-09-30", - "shared_gallery_images": "2020-09-30", - "snapshots": "2020-12-01", - "ssh_public_keys": "2020-12-01", - "usage": "2020-12-01", - "virtual_machine_extension_images": "2020-12-01", - "virtual_machine_extensions": "2020-12-01", - "virtual_machine_images": "2020-12-01", - "virtual_machine_images_edge_zone": "2020-12-01", - "virtual_machine_run_commands": "2020-12-01", - "virtual_machine_scale_set_extensions": "2020-12-01", - "virtual_machine_scale_set_rolling_upgrades": "2020-12-01", - "virtual_machine_scale_set_vm_extensions": "2020-12-01", - "virtual_machine_scale_set_vm_run_commands": "2020-12-01", - "virtual_machine_scale_set_vms": "2020-12-01", - "virtual_machine_scale_sets": "2020-12-01", - "virtual_machine_sizes": "2020-12-01", - "virtual_machines": "2020-12-01", - } - }, - AZURE_COMPUTE_MGMT_PROFILE_TAG + " osm", - ) - - def __init__(self, vim_account: Dict): - self.vim_account = vim_account - self.reload_client = True - logger = logging.getLogger("azure") - logger.setLevel(logging.ERROR) - # Store config to create azure subscription later - self._config = { - "user": vim_account["vim_user"], - "passwd": vim_account["vim_password"], - "tenant": vim_account["vim_tenant_name"], - } - - # SUBSCRIPTION - config = vim_account["config"] - if "subscription_id" in config: - self._config["subscription_id"] = config.get("subscription_id") - log.info("Subscription: %s", self._config["subscription_id"]) - else: - log.error("Subscription not specified") - return - - # RESOURCE_GROUP - if "resource_group" in config: - self.resource_group = config.get("resource_group") - else: - log.error("Azure resource_group is not specified at config") - return - - def _reload_connection(self): - if self.reload_client: - log.debug("reloading azure client") - try: - self.credentials = ClientSecretCredential( - client_id=self._config["user"], - client_secret=self._config["passwd"], - tenant_id=self._config["tenant"], - ) - self.conn_compute = ComputeManagementClient( - self.credentials, - self._config["subscription_id"], - profile=self.AZURE_COMPUTE_MGMT_PROFILE, - ) - # Set to client created - self.reload_client = False - except Exception as e: - log.error(e) - - def collect_servers_status(self) -> List[Dict]: - servers = [] - log.debug("collect_servers_status") - self._reload_connection() - try: - for vm in self.conn_compute.virtual_machines.list(self.resource_group): - id = vm.id - array = id.split("/") - name = array[-1] - status = self.provision_state2osm.get(vm.provisioning_state, "OTHER") - if vm.provisioning_state == "Succeeded": - # check if machine is running or stopped - instance_view = self.conn_compute.virtual_machines.instance_view( - self.resource_group, name - ) - for status in instance_view.statuses: - splitted_status = status.code.split("/") - if ( - len(splitted_status) == 2 - and splitted_status[0] == "PowerState" - ): - status = self.power_state2osm.get( - splitted_status[1], "OTHER" - ) - # log.info(f'id: {id}, name: {name}, status: {status}') - vm = { - "id": id, - "name": name, - "status": (1 if (status == "ACTIVE") else 0), - } - servers.append(vm) - except Exception as e: - log.error(e) - return servers - - def is_vim_ok(self) -> bool: - status = False - self.reload_client = True - try: - self._reload_connection() - status = True - except Exception as e: - log.error(e) - return status diff --git a/src/osm_mon/vim_connectors/base_vim.py b/src/osm_mon/vim_connectors/base_vim.py deleted file mode 100644 index 5eb146b..0000000 --- a/src/osm_mon/vim_connectors/base_vim.py +++ /dev/null @@ -1,29 +0,0 @@ -####################################################################################### -# Copyright ETSI Contributors and Others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################################### -from typing import Dict, List - - -class VIMConnector: - def __init__(self, vim_account: Dict): - pass - - # def collect_servers_status(self) -> List[Metric]: - def collect_servers_status(self) -> List: - pass - - def is_vim_ok(self) -> bool: - pass diff --git a/src/osm_mon/vim_connectors/gcp.py b/src/osm_mon/vim_connectors/gcp.py deleted file mode 100644 index 396e136..0000000 --- a/src/osm_mon/vim_connectors/gcp.py +++ /dev/null @@ -1,103 +0,0 @@ -####################################################################################### -# Copyright ETSI Contributors and Others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################################### -# pylint: disable=E1101 - -import logging -from typing import Dict, List - -from google.oauth2 import service_account -import googleapiclient.discovery -from osm_mon.vim_connectors.base_vim import VIMConnector - -log = logging.getLogger(__name__) - - -class GcpCollector(VIMConnector): - def __init__(self, vim_account: Dict): - self.vim_account = vim_account - self.project = vim_account["vim_tenant_name"] or vim_account["vim_tenant_id"] - - # REGION - Google Cloud considers regions and zones. A specific region - # can have more than one zone (for instance: region us-west1 with the - # zones us-west1-a, us-west1-b and us-west1-c). So the region name - # specified in the config will be considered as a specific zone for GC - # and the region will be calculated from that without the preffix. - if "config" in vim_account: - config = vim_account["config"] - if "region_name" in config: - self.zone = config.get("region_name") - self.region = self.zone.rsplit("-", 1)[0] - else: - log.error("Google Cloud region_name not specified in config") - else: - log.error("config is not specified in VIM") - - # Credentials - scopes = ["https://www.googleapis.com/auth/cloud-platform"] - self.credentials = None - if "credentials" in config: - log.debug("Setting credentials") - # Settings Google Cloud credentials dict - creds_body = config["credentials"] - creds = service_account.Credentials.from_service_account_info(creds_body) - if "sa_file" in config: - creds = service_account.Credentials.from_service_account_file( - config.get("sa_file"), scopes=scopes - ) - log.debug("Credentials: %s", creds) - # Construct a Resource for interacting with an API. - self.credentials = creds - try: - self.conn_compute = googleapiclient.discovery.build( - "compute", "v1", credentials=creds - ) - except Exception as e: - log.error(e) - else: - log.error("It is not possible to init GCP with no credentials") - - def collect_servers_status(self) -> List[Dict]: - servers = [] - try: - response = ( - self.conn_compute.instances() - .list(project=self.project, zone=self.zone) - .execute() - ) - if "items" in response: - log.info(response["items"]) - for server in response["items"]: - vm = { - "id": server["id"], - "name": server["name"], - "status": (1 if (server["status"] == "RUNNING") else 0), - } - servers.append(vm) - except Exception as e: - log.error(e) - return servers - - def is_vim_ok(self) -> bool: - status = False - try: - self.conn_compute.zones().get( - project=self.project, zone=self.zone - ).execute() - status = True - except Exception as e: - log.error(e) - return status diff --git a/src/osm_mon/vim_connectors/openstack.py b/src/osm_mon/vim_connectors/openstack.py deleted file mode 100644 index d37973d..0000000 --- a/src/osm_mon/vim_connectors/openstack.py +++ /dev/null @@ -1,91 +0,0 @@ -####################################################################################### -# Copyright ETSI Contributors and Others. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -####################################################################################### -import logging -from typing import Dict, List - -from keystoneauth1 import session -from keystoneauth1.identity import v3 -from novaclient import client as nova_client -from osm_mon.vim_connectors.base_vim import VIMConnector - -log = logging.getLogger(__name__) - - -class CertificateNotCreated(Exception): - pass - - -class OpenStackCollector(VIMConnector): - def __init__(self, vim_account: Dict): - log.info("__init__") - self.vim_account = vim_account - self.vim_session = None - self.vim_session = self._get_session(vim_account) - self.nova = self._build_nova_client() - - def _get_session(self, creds: Dict): - verify_ssl = True - project_domain_name = "Default" - user_domain_name = "Default" - try: - if "config" in creds: - vim_config = creds["config"] - if "insecure" in vim_config and vim_config["insecure"]: - verify_ssl = False - if "ca_cert" in vim_config: - verify_ssl = vim_config["ca_cert"] - elif "ca_cert_content" in vim_config: - # vim_config = self._create_file_cert(vim_config, creds["_id"]) - verify_ssl = vim_config["ca_cert"] - if "project_domain_name" in vim_config: - project_domain_name = vim_config["project_domain_name"] - if "user_domain_name" in vim_config: - user_domain_name = vim_config["user_domain_name"] - auth = v3.Password( - auth_url=creds["vim_url"], - username=creds["vim_user"], - password=creds["vim_password"], - project_name=creds["vim_tenant_name"], - project_domain_name=project_domain_name, - user_domain_name=user_domain_name, - ) - return session.Session(auth=auth, verify=verify_ssl, timeout=10) - except CertificateNotCreated as e: - log.error(e) - - def _build_nova_client(self) -> nova_client.Client: - return nova_client.Client("2", session=self.vim_session, timeout=10) - - def collect_servers_status(self) -> List[Dict]: - log.info("collect_servers_status") - servers = [] - for server in self.nova.servers.list(detailed=True): - vm = { - "id": server.id, - "name": server.name, - "status": (0 if (server.status == "ERROR") else 1), - } - servers.append(vm) - return servers - - def is_vim_ok(self) -> bool: - try: - self.nova.servers.list() - return True - except Exception as e: - log.warning("VIM status is not OK: %s" % e) - return False diff --git a/src/osm_ngsa/__init__.py b/src/osm_ngsa/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/osm_ngsa/dags/.airflowignore b/src/osm_ngsa/dags/.airflowignore new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/osm_ngsa/dags/.airflowignore @@ -0,0 +1 @@ + diff --git a/src/osm_ngsa/dags/__init__.py b/src/osm_ngsa/dags/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/osm_ngsa/dags/multivim_vim_status.py b/src/osm_ngsa/dags/multivim_vim_status.py new file mode 100644 index 0000000..93894b1 --- /dev/null +++ b/src/osm_ngsa/dags/multivim_vim_status.py @@ -0,0 +1,153 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.decorators import task +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +from osm_mon.vim_connectors.azure import AzureCollector +from osm_mon.vim_connectors.gcp import GcpCollector +from osm_mon.vim_connectors.openstack import OpenStackCollector +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway + + +SUPPORTED_VIM_TYPES = ["openstack", "vio", "gcp", "azure"] +PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091" +PROMETHEUS_JOB_PREFIX = "airflow_osm_vim_status_" +PROMETHEUS_METRIC = "vim_status" +PROMETHEUS_METRIC_DESCRIPTION = "VIM status" +SCHEDULE_INTERVAL = 1 + + +def get_all_vim(): + """Get VIMs from MongoDB""" + print("Getting VIM list") + + cfg = Config() + print(cfg.conf) + common_db = CommonDbClient(cfg) + vim_accounts = common_db.get_vim_accounts() + vim_list = [] + for vim in vim_accounts: + print(f'Read VIM {vim["_id"]} ({vim["name"]})') + vim_list.append( + {"_id": vim["_id"], "name": vim["name"], "vim_type": vim["vim_type"]} + ) + + print(vim_list) + print("Getting VIM list OK") + return vim_list + + +def create_dag(dag_id, dag_number, dag_description, vim_id): + dag = DAG( + dag_id, + catchup=False, + default_args={ + "depends_on_past": False, + "retries": 1, + # "retry_delay": timedelta(minutes=1), + "retry_delay": timedelta(seconds=10), + }, + description=dag_description, + is_paused_upon_creation=False, + # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL), + schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *", + start_date=datetime(2022, 1, 1), + tags=["osm", "vim"], + ) + + with dag: + + def get_vim_collector(vim_account): + """Return a VIM collector for the vim_account""" + vim_type = vim_account["vim_type"] + if "config" in vim_account and "vim_type" in vim_account["config"]: + vim_type = vim_account["config"]["vim_type"].lower() + if vim_type == "vio" and "vrops_site" not in vim_account["config"]: + vim_type = "openstack" + if vim_type == "openstack": + return OpenStackCollector(vim_account) + if vim_type == "gcp": + return GcpCollector(vim_account) + if vim_type == "azure": + return AzureCollector(vim_account) + print(f"VIM type '{vim_type}' not supported") + return None + + @task(task_id="get_vim_status_and_send_to_prometheus") + def get_vim_status_and_send_to_prometheus(vim_id: str): + """Authenticate against VIM and check status""" + + # Get VIM account info from MongoDB + print(f"Reading VIM info, id: {vim_id}") + cfg = Config() + common_db = CommonDbClient(cfg) + vim_account = common_db.get_vim_account(vim_account_id=vim_id) + print(vim_account) + + # Define Prometheus Metric for NS topology + registry = CollectorRegistry() + metric = Gauge( + PROMETHEUS_METRIC, + PROMETHEUS_METRIC_DESCRIPTION, + labelnames=[ + "vim_id", + ], + registry=registry, + ) + metric.labels(vim_id).set(0) + + # Get status of VIM + collector = get_vim_collector(vim_account) + if collector: + status = collector.is_vim_ok() + print(f"VIM status: {status}") + metric.labels(vim_id).set(1) + else: + print("Error creating VIM collector") + # Push to Prometheus + push_to_gateway( + gateway=PROMETHEUS_PUSHGW, + job=f"{PROMETHEUS_JOB_PREFIX}{vim_id}", + registry=registry, + ) + return + + get_vim_status_and_send_to_prometheus(vim_id) + + return dag + + +vim_list = get_all_vim() +for index, vim in enumerate(vim_list): + vim_type = vim["vim_type"] + if vim_type in SUPPORTED_VIM_TYPES: + vim_id = vim["_id"] + vim_name = vim["name"] + dag_description = f"Dag for VIM {vim_name} status" + dag_id = f"vim_status_{vim_id}" + print(f"Creating DAG {dag_id}") + globals()[dag_id] = create_dag( + dag_id=dag_id, + dag_number=index, + dag_description=dag_description, + vim_id=vim_id, + ) + else: + print(f"VIM type '{vim_type}' not supported for monitoring VIM status") diff --git a/src/osm_ngsa/dags/multivim_vm_status.py b/src/osm_ngsa/dags/multivim_vm_status.py new file mode 100644 index 0000000..dbdbbc0 --- /dev/null +++ b/src/osm_ngsa/dags/multivim_vm_status.py @@ -0,0 +1,166 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.decorators import task +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +from osm_mon.vim_connectors.azure import AzureCollector +from osm_mon.vim_connectors.gcp import GcpCollector +from osm_mon.vim_connectors.openstack import OpenStackCollector +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway + + +SUPPORTED_VIM_TYPES = ["openstack", "vio", "gcp", "azure"] +PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091" +PROMETHEUS_JOB_PREFIX = "airflow_osm_vm_status_" +PROMETHEUS_METRIC = "vm_status" +PROMETHEUS_METRIC_DESCRIPTION = "VM Status from VIM" +SCHEDULE_INTERVAL = 1 + + +def get_all_vim(): + """Get VIMs from MongoDB""" + print("Getting VIM list") + + cfg = Config() + print(cfg.conf) + common_db = CommonDbClient(cfg) + vim_accounts = common_db.get_vim_accounts() + vim_list = [] + for vim in vim_accounts: + print(f'Read VIM {vim["_id"]} ({vim["name"]})') + vim_list.append( + {"_id": vim["_id"], "name": vim["name"], "vim_type": vim["vim_type"]} + ) + + print(vim_list) + print("Getting VIM list OK") + return vim_list + + +def create_dag(dag_id, dag_number, dag_description, vim_id): + dag = DAG( + dag_id, + catchup=False, + default_args={ + "depends_on_past": False, + "retries": 1, + # "retry_delay": timedelta(minutes=1), + "retry_delay": timedelta(seconds=10), + }, + description=dag_description, + is_paused_upon_creation=False, + # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL), + schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *", + start_date=datetime(2022, 1, 1), + tags=["osm", "vim"], + ) + + with dag: + + def get_vim_collector(vim_account): + """Return a VIM collector for the vim_account""" + vim_type = vim_account["vim_type"] + if "config" in vim_account and "vim_type" in vim_account["config"]: + vim_type = vim_account["config"]["vim_type"].lower() + if vim_type == "vio" and "vrops_site" not in vim_account["config"]: + vim_type = "openstack" + if vim_type == "openstack": + return OpenStackCollector(vim_account) + if vim_type == "gcp": + return GcpCollector(vim_account) + if vim_type == "azure": + return AzureCollector(vim_account) + print(f"VIM type '{vim_type}' not supported") + return None + + def get_all_vm_status(vim_account): + """Get VM status from the VIM""" + collector = get_vim_collector(vim_account) + if collector: + status = collector.is_vim_ok() + print(f"VIM status: {status}") + vm_status_list = collector.collect_servers_status() + return vm_status_list + else: + return None + + @task(task_id="get_all_vm_status_and_send_to_prometheus") + def get_all_vm_status_and_send_to_prometheus(vim_id: str): + """Authenticate against VIM, collect servers status and send to prometheus""" + + # Get VIM account info from MongoDB + print(f"Reading VIM info, id: {vim_id}") + cfg = Config() + common_db = CommonDbClient(cfg) + vim_account = common_db.get_vim_account(vim_account_id=vim_id) + print(vim_account) + + # Define Prometheus Metric for NS topology + registry = CollectorRegistry() + metric = Gauge( + PROMETHEUS_METRIC, + PROMETHEUS_METRIC_DESCRIPTION, + labelnames=[ + "vm_id", + "vim_id", + ], + registry=registry, + ) + + # Get status of all VM from VIM + all_vm_status = get_all_vm_status(vim_account) + print(f"Got {len(all_vm_status)} VMs with their status:") + if all_vm_status: + for vm in all_vm_status: + vm_id = vm["id"] + vm_status = vm["status"] + vm_name = vm.get("name", "") + print(f" {vm_name} ({vm_id}) {vm_status}") + metric.labels(vm_id, vim_id).set(vm_status) + # Push to Prometheus only if there are VM + push_to_gateway( + gateway=PROMETHEUS_PUSHGW, + job=f"{PROMETHEUS_JOB_PREFIX}{vim_id}", + registry=registry, + ) + return + + get_all_vm_status_and_send_to_prometheus(vim_id) + + return dag + + +vim_list = get_all_vim() +for index, vim in enumerate(vim_list): + vim_type = vim["vim_type"] + if vim_type in SUPPORTED_VIM_TYPES: + vim_id = vim["_id"] + vim_name = vim["name"] + dag_description = f"Dag for vim {vim_name}" + dag_id = f"vm_status_vim_{vim_id}" + print(f"Creating DAG {dag_id}") + globals()[dag_id] = create_dag( + dag_id=dag_id, + dag_number=index, + dag_description=dag_description, + vim_id=vim_id, + ) + else: + print(f"VIM type '{vim_type}' not supported for collecting VM status") diff --git a/src/osm_ngsa/dags/ns_topology.py b/src/osm_ngsa/dags/ns_topology.py new file mode 100644 index 0000000..d3fb504 --- /dev/null +++ b/src/osm_ngsa/dags/ns_topology.py @@ -0,0 +1,156 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +from datetime import datetime, timedelta + +from airflow.decorators import dag, task +from osm_mon.core.common_db import CommonDbClient +from osm_mon.core.config import Config +from prometheus_client import CollectorRegistry, Gauge, push_to_gateway + + +PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091" +PROMETHEUS_JOB = "airflow_osm_ns_topology" +PROMETHEUS_METRIC = "ns_topology" +PROMETHEUS_METRIC_DESCRIPTION = "Network services topology" +SCHEDULE_INTERVAL = 2 + + +@dag( + catchup=False, + default_args={ + "depends_on_past": False, + "retries": 1, + "retry_delay": timedelta(seconds=10), + }, + description="NS topology", + is_paused_upon_creation=False, + # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL), + schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *", + start_date=datetime(2022, 1, 1), + tags=["osm", "topology"], +) +def ns_topology(): + @task(task_id="get_topology") + def get_topology(): + """ + Get NS topology from MongoDB and exports it as a metric + to Prometheus + """ + + # Define Prometheus Metric for NS topology + registry = CollectorRegistry() + metric = Gauge( + PROMETHEUS_METRIC, + PROMETHEUS_METRIC_DESCRIPTION, + labelnames=[ + "ns_id", + "project_id", + "vnf_id", + "vdu_id", + "vm_id", + "vim_id", + "vdu_name", + "vnf_member_index", + ], + registry=registry, + ) + + # Getting VNFR list from MongoDB + print("Getting VNFR list from MongoDB") + cfg = Config() + print(cfg.conf) + common_db = CommonDbClient(cfg) + vnfr_list = common_db.get_vnfrs() + + # Only send topology if ns state is one of the nsAllowedStatesSet + nsAllowedStatesSet = {"INSTANTIATED"} + + # For loop to get NS topology. + # For each VDU, a metric sample is created with the appropriate labels + for vnfr in vnfr_list: + # Label ns_id + vnf_id = vnfr["_id"] + # Label ns_id + ns_id = vnfr["nsr-id-ref"] + # Label vnfd_id + vnfd_id = vnfr["vnfd-ref"] + # Label project_id + project_list = vnfr.get("_admin", {}).get("projects_read", []) + project_id = "None" + if project_list: + project_id = project_list[0] + # TODO: use logger with loglevels instead of print + # Other info + ns_state = vnfr["_admin"]["nsState"] + vnf_membex_index = vnfr["member-vnf-index-ref"] + print( + f"Read VNFR: id: {vnf_id}, ns_id: {ns_id}, ", + f"state: {ns_state}, vnfd_id: {vnfd_id}, ", + f"vnf_membex_index: {vnf_membex_index}, ", + f"project_id: {project_id}", + ) + # Only send topology if ns State is one of the nsAllowedStatesSet + if ns_state not in nsAllowedStatesSet: + continue + + print("VDU list:") + for vdu in vnfr.get("vdur", []): + # Label vdu_id + vdu_id = vdu["_id"] + # Label vim_id + vim_info = vdu.get("vim_info") + if not vim_info: + print("Error: vim_info not available in vdur") + continue + if len(vim_info) != 1: + print("Error: more than one vim_info in vdur") + continue + vim_id = next(iter(vim_info))[4:] + # Label vm_id + vm_id = vdu["vim-id"] + # Other VDU info + vdu_name = vdu.get("name", "UNKNOWN") + print( + f" id: {vdu_id}, name: {vdu_name}, " + f"vim_id: {vim_id}, vm_id: {vm_id}" + ) + print( + f"METRIC SAMPLE: ns_id: {ns_id}, ", + f"project_id: {project_id}, vnf_id: {vnf_id}, ", + f"vdu_id: {vdu_id}, vm_id: {vm_id}, vim_id: {vim_id}", + ) + metric.labels( + ns_id, + project_id, + vnf_id, + vdu_id, + vm_id, + vim_id, + vdu_name, + vnf_membex_index, + ).set(1) + + # print("Push to gateway") + push_to_gateway( + gateway=PROMETHEUS_PUSHGW, job=PROMETHEUS_JOB, registry=registry + ) + return + + get_topology() + + +dag = ns_topology() diff --git a/src/osm_ngsa/osm_mon/__init__.py b/src/osm_ngsa/osm_mon/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/osm_ngsa/osm_mon/core/__init__.py b/src/osm_ngsa/osm_mon/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/osm_ngsa/osm_mon/core/common_db.py b/src/osm_ngsa/osm_mon/core/common_db.py new file mode 100644 index 0000000..7c579c3 --- /dev/null +++ b/src/osm_ngsa/osm_mon/core/common_db.py @@ -0,0 +1,90 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +from osm_common import dbmemory, dbmongo +from osm_mon.core.config import Config + + +class CommonDbClient: + def __init__(self, config: Config): + if config.get("database", "driver") == "mongo": + self.common_db = dbmongo.DbMongo() + elif config.get("database", "driver") == "memory": + self.common_db = dbmemory.DbMemory() + else: + raise Exception( + "Unknown database driver {}".format(config.get("section", "driver")) + ) + self.common_db.db_connect(config.get("database")) + + def get_vnfr(self, nsr_id: str, member_index: int): + vnfr = self.common_db.get_one( + "vnfrs", {"nsr-id-ref": nsr_id, "member-vnf-index-ref": str(member_index)} + ) + return vnfr + + def get_vnfrs(self, nsr_id: str = None, vim_account_id: str = None): + if nsr_id and vim_account_id: + raise NotImplementedError("Only one filter is currently supported") + if nsr_id: + vnfrs = [ + self.get_vnfr(nsr_id, member["member-vnf-index"]) + for member in self.get_nsr(nsr_id)["nsd"]["constituent-vnfd"] + ] + elif vim_account_id: + vnfrs = self.common_db.get_list("vnfrs", {"vim-account-id": vim_account_id}) + else: + vnfrs = self.common_db.get_list("vnfrs") + return vnfrs + + def get_nsr(self, nsr_id: str): + nsr = self.common_db.get_one("nsrs", {"id": nsr_id}) + return nsr + + def decrypt_vim_password(self, vim_password: str, schema_version: str, vim_id: str): + return self.common_db.decrypt(vim_password, schema_version, vim_id) + + def get_vim_accounts(self): + return self.common_db.get_list("vim_accounts") + + def get_vim_account(self, vim_account_id: str) -> dict: + vim_account = self.common_db.get_one("vim_accounts", {"_id": vim_account_id}) + vim_account["vim_password"] = self.decrypt_vim_password( + vim_account["vim_password"], vim_account["schema_version"], vim_account_id + ) + vim_config_encrypted_dict = { + "1.1": ("admin_password", "nsx_password", "vcenter_password"), + "default": ( + "admin_password", + "nsx_password", + "vcenter_password", + "vrops_password", + ), + } + vim_config_encrypted = vim_config_encrypted_dict["default"] + if vim_account["schema_version"] in vim_config_encrypted_dict.keys(): + vim_config_encrypted = vim_config_encrypted_dict[ + vim_account["schema_version"] + ] + if "config" in vim_account: + for key in vim_account["config"]: + if key in vim_config_encrypted: + vim_account["config"][key] = self.decrypt_vim_password( + vim_account["config"][key], + vim_account["schema_version"], + vim_account_id, + ) + return vim_account diff --git a/src/osm_ngsa/osm_mon/core/config.py b/src/osm_ngsa/osm_mon/core/config.py new file mode 100644 index 0000000..eb0e493 --- /dev/null +++ b/src/osm_ngsa/osm_mon/core/config.py @@ -0,0 +1,68 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +"""Global configuration managed by environment variables.""" + +import logging +import os + +import pkg_resources +import yaml + +logger = logging.getLogger(__name__) + + +class Config: + def __init__(self, config_file: str = ""): + self.conf = {} + self._read_config_file(config_file) + self._read_env() + + def _read_config_file(self, config_file): + if not config_file: + path = "config.yaml" + config_file = pkg_resources.resource_filename(__name__, path) + with open(config_file) as f: + self.conf = yaml.load(f) + + def get(self, section, field=None): + if not field: + return self.conf[section] + return self.conf[section].get(field) + + def set(self, section, field, value): + if section not in self.conf: + self.conf[section] = {} + self.conf[section][field] = value + + def _read_env(self): + for env in os.environ: + if not env.startswith("OSMMON_"): + continue + elements = env.lower().split("_") + if len(elements) < 3: + logger.warning( + "Environment variable %s=%s does not comply with required format. Section and/or field missing.", + env, + os.getenv(env), + ) + continue + section = elements[1] + field = "_".join(elements[2:]) + value = os.getenv(env) + if section not in self.conf: + self.conf[section] = {} + self.conf[section][field] = value diff --git a/src/osm_ngsa/osm_mon/core/config.yaml b/src/osm_ngsa/osm_mon/core/config.yaml new file mode 100644 index 0000000..197c818 --- /dev/null +++ b/src/osm_ngsa/osm_mon/core/config.yaml @@ -0,0 +1,27 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +global: + loglevel: INFO + request_timeout: 10 + +database: + driver: mongo + uri: mongodb://mongodb-k8s:27017/?replicaSet=rs0 + name: osm + commonkey: gj7LmbCexbmII7memwbGRRdfbYuT3nvy + diff --git a/src/osm_ngsa/osm_mon/vim_connectors/__init__.py b/src/osm_ngsa/osm_mon/vim_connectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/osm_ngsa/osm_mon/vim_connectors/azure.py b/src/osm_ngsa/osm_mon/vim_connectors/azure.py new file mode 100644 index 0000000..23086c4 --- /dev/null +++ b/src/osm_ngsa/osm_mon/vim_connectors/azure.py @@ -0,0 +1,193 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +import logging +from typing import Dict, List + +from azure.identity import ClientSecretCredential +from azure.mgmt.compute import ComputeManagementClient +from azure.profiles import ProfileDefinition +from osm_mon.vim_connectors.base_vim import VIMConnector + + +log = logging.getLogger(__name__) + + +class AzureCollector(VIMConnector): + + # Translate azure provisioning state to OSM provision state. + # The first three ones are the transitional status once a user initiated + # action has been requested. Once the operation is complete, it will + # transition into the states Succeeded or Failed + # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/states-lifecycle + provision_state2osm = { + "Creating": "BUILD", + "Updating": "BUILD", + "Deleting": "INACTIVE", + "Succeeded": "ACTIVE", + "Failed": "ERROR", + } + + # Translate azure power state to OSM provision state + power_state2osm = { + "starting": "INACTIVE", + "running": "ACTIVE", + "stopping": "INACTIVE", + "stopped": "INACTIVE", + "unknown": "OTHER", + "deallocated": "BUILD", + "deallocating": "BUILD", + } + + AZURE_COMPUTE_MGMT_CLIENT_API_VERSION = "2021-03-01" + AZURE_COMPUTE_MGMT_PROFILE_TAG = "azure.mgmt.compute.ComputeManagementClient" + AZURE_COMPUTE_MGMT_PROFILE = ProfileDefinition( + { + AZURE_COMPUTE_MGMT_PROFILE_TAG: { + None: AZURE_COMPUTE_MGMT_CLIENT_API_VERSION, + "availability_sets": "2020-12-01", + "dedicated_host_groups": "2020-12-01", + "dedicated_hosts": "2020-12-01", + "disk_accesses": "2020-12-01", + "disk_encryption_sets": "2020-12-01", + "disk_restore_point": "2020-12-01", + "disks": "2020-12-01", + "galleries": "2020-09-30", + "gallery_application_versions": "2020-09-30", + "gallery_applications": "2020-09-30", + "gallery_image_versions": "2020-09-30", + "gallery_images": "2020-09-30", + "gallery_sharing_profile": "2020-09-30", + "images": "2020-12-01", + "log_analytics": "2020-12-01", + "operations": "2020-12-01", + "proximity_placement_groups": "2020-12-01", + "resource_skus": "2019-04-01", + "shared_galleries": "2020-09-30", + "shared_gallery_image_versions": "2020-09-30", + "shared_gallery_images": "2020-09-30", + "snapshots": "2020-12-01", + "ssh_public_keys": "2020-12-01", + "usage": "2020-12-01", + "virtual_machine_extension_images": "2020-12-01", + "virtual_machine_extensions": "2020-12-01", + "virtual_machine_images": "2020-12-01", + "virtual_machine_images_edge_zone": "2020-12-01", + "virtual_machine_run_commands": "2020-12-01", + "virtual_machine_scale_set_extensions": "2020-12-01", + "virtual_machine_scale_set_rolling_upgrades": "2020-12-01", + "virtual_machine_scale_set_vm_extensions": "2020-12-01", + "virtual_machine_scale_set_vm_run_commands": "2020-12-01", + "virtual_machine_scale_set_vms": "2020-12-01", + "virtual_machine_scale_sets": "2020-12-01", + "virtual_machine_sizes": "2020-12-01", + "virtual_machines": "2020-12-01", + } + }, + AZURE_COMPUTE_MGMT_PROFILE_TAG + " osm", + ) + + def __init__(self, vim_account: Dict): + self.vim_account = vim_account + self.reload_client = True + logger = logging.getLogger("azure") + logger.setLevel(logging.ERROR) + # Store config to create azure subscription later + self._config = { + "user": vim_account["vim_user"], + "passwd": vim_account["vim_password"], + "tenant": vim_account["vim_tenant_name"], + } + + # SUBSCRIPTION + config = vim_account["config"] + if "subscription_id" in config: + self._config["subscription_id"] = config.get("subscription_id") + log.info("Subscription: %s", self._config["subscription_id"]) + else: + log.error("Subscription not specified") + return + + # RESOURCE_GROUP + if "resource_group" in config: + self.resource_group = config.get("resource_group") + else: + log.error("Azure resource_group is not specified at config") + return + + def _reload_connection(self): + if self.reload_client: + log.debug("reloading azure client") + try: + self.credentials = ClientSecretCredential( + client_id=self._config["user"], + client_secret=self._config["passwd"], + tenant_id=self._config["tenant"], + ) + self.conn_compute = ComputeManagementClient( + self.credentials, + self._config["subscription_id"], + profile=self.AZURE_COMPUTE_MGMT_PROFILE, + ) + # Set to client created + self.reload_client = False + except Exception as e: + log.error(e) + + def collect_servers_status(self) -> List[Dict]: + servers = [] + log.debug("collect_servers_status") + self._reload_connection() + try: + for vm in self.conn_compute.virtual_machines.list(self.resource_group): + id = vm.id + array = id.split("/") + name = array[-1] + status = self.provision_state2osm.get(vm.provisioning_state, "OTHER") + if vm.provisioning_state == "Succeeded": + # check if machine is running or stopped + instance_view = self.conn_compute.virtual_machines.instance_view( + self.resource_group, name + ) + for status in instance_view.statuses: + splitted_status = status.code.split("/") + if ( + len(splitted_status) == 2 + and splitted_status[0] == "PowerState" + ): + status = self.power_state2osm.get( + splitted_status[1], "OTHER" + ) + # log.info(f'id: {id}, name: {name}, status: {status}') + vm = { + "id": id, + "name": name, + "status": (1 if (status == "ACTIVE") else 0), + } + servers.append(vm) + except Exception as e: + log.error(e) + return servers + + def is_vim_ok(self) -> bool: + status = False + self.reload_client = True + try: + self._reload_connection() + status = True + except Exception as e: + log.error(e) + return status diff --git a/src/osm_ngsa/osm_mon/vim_connectors/base_vim.py b/src/osm_ngsa/osm_mon/vim_connectors/base_vim.py new file mode 100644 index 0000000..5eb146b --- /dev/null +++ b/src/osm_ngsa/osm_mon/vim_connectors/base_vim.py @@ -0,0 +1,29 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +from typing import Dict, List + + +class VIMConnector: + def __init__(self, vim_account: Dict): + pass + + # def collect_servers_status(self) -> List[Metric]: + def collect_servers_status(self) -> List: + pass + + def is_vim_ok(self) -> bool: + pass diff --git a/src/osm_ngsa/osm_mon/vim_connectors/gcp.py b/src/osm_ngsa/osm_mon/vim_connectors/gcp.py new file mode 100644 index 0000000..396e136 --- /dev/null +++ b/src/osm_ngsa/osm_mon/vim_connectors/gcp.py @@ -0,0 +1,103 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +# pylint: disable=E1101 + +import logging +from typing import Dict, List + +from google.oauth2 import service_account +import googleapiclient.discovery +from osm_mon.vim_connectors.base_vim import VIMConnector + +log = logging.getLogger(__name__) + + +class GcpCollector(VIMConnector): + def __init__(self, vim_account: Dict): + self.vim_account = vim_account + self.project = vim_account["vim_tenant_name"] or vim_account["vim_tenant_id"] + + # REGION - Google Cloud considers regions and zones. A specific region + # can have more than one zone (for instance: region us-west1 with the + # zones us-west1-a, us-west1-b and us-west1-c). So the region name + # specified in the config will be considered as a specific zone for GC + # and the region will be calculated from that without the preffix. + if "config" in vim_account: + config = vim_account["config"] + if "region_name" in config: + self.zone = config.get("region_name") + self.region = self.zone.rsplit("-", 1)[0] + else: + log.error("Google Cloud region_name not specified in config") + else: + log.error("config is not specified in VIM") + + # Credentials + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + self.credentials = None + if "credentials" in config: + log.debug("Setting credentials") + # Settings Google Cloud credentials dict + creds_body = config["credentials"] + creds = service_account.Credentials.from_service_account_info(creds_body) + if "sa_file" in config: + creds = service_account.Credentials.from_service_account_file( + config.get("sa_file"), scopes=scopes + ) + log.debug("Credentials: %s", creds) + # Construct a Resource for interacting with an API. + self.credentials = creds + try: + self.conn_compute = googleapiclient.discovery.build( + "compute", "v1", credentials=creds + ) + except Exception as e: + log.error(e) + else: + log.error("It is not possible to init GCP with no credentials") + + def collect_servers_status(self) -> List[Dict]: + servers = [] + try: + response = ( + self.conn_compute.instances() + .list(project=self.project, zone=self.zone) + .execute() + ) + if "items" in response: + log.info(response["items"]) + for server in response["items"]: + vm = { + "id": server["id"], + "name": server["name"], + "status": (1 if (server["status"] == "RUNNING") else 0), + } + servers.append(vm) + except Exception as e: + log.error(e) + return servers + + def is_vim_ok(self) -> bool: + status = False + try: + self.conn_compute.zones().get( + project=self.project, zone=self.zone + ).execute() + status = True + except Exception as e: + log.error(e) + return status diff --git a/src/osm_ngsa/osm_mon/vim_connectors/openstack.py b/src/osm_ngsa/osm_mon/vim_connectors/openstack.py new file mode 100644 index 0000000..d37973d --- /dev/null +++ b/src/osm_ngsa/osm_mon/vim_connectors/openstack.py @@ -0,0 +1,91 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +import logging +from typing import Dict, List + +from keystoneauth1 import session +from keystoneauth1.identity import v3 +from novaclient import client as nova_client +from osm_mon.vim_connectors.base_vim import VIMConnector + +log = logging.getLogger(__name__) + + +class CertificateNotCreated(Exception): + pass + + +class OpenStackCollector(VIMConnector): + def __init__(self, vim_account: Dict): + log.info("__init__") + self.vim_account = vim_account + self.vim_session = None + self.vim_session = self._get_session(vim_account) + self.nova = self._build_nova_client() + + def _get_session(self, creds: Dict): + verify_ssl = True + project_domain_name = "Default" + user_domain_name = "Default" + try: + if "config" in creds: + vim_config = creds["config"] + if "insecure" in vim_config and vim_config["insecure"]: + verify_ssl = False + if "ca_cert" in vim_config: + verify_ssl = vim_config["ca_cert"] + elif "ca_cert_content" in vim_config: + # vim_config = self._create_file_cert(vim_config, creds["_id"]) + verify_ssl = vim_config["ca_cert"] + if "project_domain_name" in vim_config: + project_domain_name = vim_config["project_domain_name"] + if "user_domain_name" in vim_config: + user_domain_name = vim_config["user_domain_name"] + auth = v3.Password( + auth_url=creds["vim_url"], + username=creds["vim_user"], + password=creds["vim_password"], + project_name=creds["vim_tenant_name"], + project_domain_name=project_domain_name, + user_domain_name=user_domain_name, + ) + return session.Session(auth=auth, verify=verify_ssl, timeout=10) + except CertificateNotCreated as e: + log.error(e) + + def _build_nova_client(self) -> nova_client.Client: + return nova_client.Client("2", session=self.vim_session, timeout=10) + + def collect_servers_status(self) -> List[Dict]: + log.info("collect_servers_status") + servers = [] + for server in self.nova.servers.list(detailed=True): + vm = { + "id": server.id, + "name": server.name, + "status": (0 if (server.status == "ERROR") else 1), + } + servers.append(vm) + return servers + + def is_vim_ok(self) -> bool: + try: + self.nova.servers.list() + return True + except Exception as e: + log.warning("VIM status is not OK: %s" % e) + return False diff --git a/tox.ini b/tox.ini index 7247cf3..4a9dcbf 100644 --- a/tox.ini +++ b/tox.ini @@ -103,6 +103,23 @@ commands = done" +####################################################################################### +[testenv:dist] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dist.txt + +# In the commands, we copy the requirements.txt to be presented as a source file (.py) +# so it gets included in the .deb package for others to consume +commands = + sh -c 'cp requirements.txt src/osm_ngsa/requirements.txt' + sh -c 'cp README.rst src/osm_ngsa/README.rst' + python3 setup.py --command-packages=stdeb.command sdist_dsc + sh -c 'cd deb_dist/osm-ngsa*/ && dpkg-buildpackage -rfakeroot -uc -us' + sh -c 'rm src/osm_ngsa/requirements.txt' + sh -c 'rm src/osm_ngsa/README.rst' +whitelist_externals = sh + + ####################################################################################### [flake8] ignore =