*.local
local
+src/osm_ngsa/_version.py
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+
+include src/osm_ngsa/requirements.txt
+include src/osm_ngsa/README.rst
+
--- /dev/null
+..
+ #######################################################################################
+ # Copyright ETSI Contributors and Others.
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ # implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #######################################################################################
+
+===========
+osm-ngsa
+===========
+
+osm-ngsa is the new Service Assurance Python library for OSM. It includes the different Airflow DAGs used for specific monitoring and alerting stages, as well as common libraries used by those DAGs to interact with VIM.
+
# limitations under the License.
#
-echo "Nothing to be done"
+MDG=NG-SA
+rm -rf pool
+rm -rf dists
+mkdir -p pool/$MDG
+mv deb_dist/*.deb pool/$MDG/
+mkdir -p dists/unstable/$MDG/binary-amd64/
+apt-ftparchive packages pool/$MDG > dists/unstable/$MDG/binary-amd64/Packages
+gzip -9fk dists/unstable/$MDG/binary-amd64/Packages
+echo "dists/**,pool/$MDG/*.deb"
+
# limitations under the License.
#
-echo "Nothing to be done"
+rm -rf dist deb_dist osm*.tar.gz *.egg-info .eggs
+
+tox -e dist
# flask-babel
# python-nvd3
# swagger-ui-bundle
-jsonschema==4.17.0
+jsonschema==4.17.1
# via
# apache-airflow
# connexion
--- /dev/null
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stdeb
+setuptools-scm
+setuptools<60
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+packaging==21.3
+ # via setuptools-scm
+pyparsing==3.0.9
+ # via packaging
+setuptools-scm==7.0.5
+ # via -r requirements-dist.in
+stdeb==0.10.0
+ # via -r requirements-dist.in
+tomli==2.0.1
+ # via setuptools-scm
+typing-extensions==4.4.0
+ # via setuptools-scm
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==59.8.0
+ # via
+ # -r requirements-dist.in
+ # setuptools-scm
--- /dev/null
+#!/usr/bin/env python3
+#
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from setuptools import setup, find_packages, find_namespace_packages
+
+_name = "osm_ngsa"
+_description = "OSM Service Assurance Airflow DAGs and libraries"
+with open(os.path.join(".", "README.rst")) as readme_file:
+ README = readme_file.read()
+
+setup(
+ name=_name,
+ description=_description,
+ long_description=README,
+ use_scm_version={
+ "write_to": "src/osm_ngsa/_version.py"
+ },
+ author="ETSI OSM",
+ author_email="osmsupport@etsi.org",
+ maintainer="ETSI OSM",
+ maintainer_email="osmsupport@etsi.org",
+ url="https://osm.etsi.org/gitweb/?p=osm/NG-SA.git;a=summary",
+ license="Apache 2.0",
+ package_dir={"": "src"},
+ packages=find_namespace_packages(where='src'),
+ include_package_data=True,
+ setup_requires=["setuptools-scm"],
+)
+
+++ /dev/null
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################
-from datetime import datetime, timedelta
-
-from airflow import DAG
-from airflow.decorators import task
-from osm_mon.core.common_db import CommonDbClient
-from osm_mon.core.config import Config
-from osm_mon.vim_connectors.azure import AzureCollector
-from osm_mon.vim_connectors.gcp import GcpCollector
-from osm_mon.vim_connectors.openstack import OpenStackCollector
-from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
-
-
-SUPPORTED_VIM_TYPES = ["openstack", "vio", "gcp", "azure"]
-PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091"
-PROMETHEUS_JOB_PREFIX = "airflow_osm_vim_status_"
-PROMETHEUS_METRIC = "vim_status"
-PROMETHEUS_METRIC_DESCRIPTION = "VIM status"
-SCHEDULE_INTERVAL = 1
-
-
-def get_all_vim():
- """Get VIMs from MongoDB"""
- print("Getting VIM list")
-
- cfg = Config()
- print(cfg.conf)
- common_db = CommonDbClient(cfg)
- vim_accounts = common_db.get_vim_accounts()
- vim_list = []
- for vim in vim_accounts:
- print(f'Read VIM {vim["_id"]} ({vim["name"]})')
- vim_list.append(
- {"_id": vim["_id"], "name": vim["name"], "vim_type": vim["vim_type"]}
- )
-
- print(vim_list)
- print("Getting VIM list OK")
- return vim_list
-
-
-def create_dag(dag_id, dag_number, dag_description, vim_id):
- dag = DAG(
- dag_id,
- catchup=False,
- default_args={
- "depends_on_past": False,
- "retries": 1,
- # "retry_delay": timedelta(minutes=1),
- "retry_delay": timedelta(seconds=10),
- },
- description=dag_description,
- is_paused_upon_creation=False,
- # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL),
- schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *",
- start_date=datetime(2022, 1, 1),
- tags=["osm", "vim"],
- )
-
- with dag:
-
- def get_vim_collector(vim_account):
- """Return a VIM collector for the vim_account"""
- vim_type = vim_account["vim_type"]
- if "config" in vim_account and "vim_type" in vim_account["config"]:
- vim_type = vim_account["config"]["vim_type"].lower()
- if vim_type == "vio" and "vrops_site" not in vim_account["config"]:
- vim_type = "openstack"
- if vim_type == "openstack":
- return OpenStackCollector(vim_account)
- if vim_type == "gcp":
- return GcpCollector(vim_account)
- if vim_type == "azure":
- return AzureCollector(vim_account)
- print(f"VIM type '{vim_type}' not supported")
- return None
-
- @task(task_id="get_vim_status_and_send_to_prometheus")
- def get_vim_status_and_send_to_prometheus(vim_id: str):
- """Authenticate against VIM and check status"""
-
- # Get VIM account info from MongoDB
- print(f"Reading VIM info, id: {vim_id}")
- cfg = Config()
- common_db = CommonDbClient(cfg)
- vim_account = common_db.get_vim_account(vim_account_id=vim_id)
- print(vim_account)
-
- # Define Prometheus Metric for NS topology
- registry = CollectorRegistry()
- metric = Gauge(
- PROMETHEUS_METRIC,
- PROMETHEUS_METRIC_DESCRIPTION,
- labelnames=[
- "vim_id",
- ],
- registry=registry,
- )
- metric.labels(vim_id).set(0)
-
- # Get status of VIM
- collector = get_vim_collector(vim_account)
- if collector:
- status = collector.is_vim_ok()
- print(f"VIM status: {status}")
- metric.labels(vim_id).set(1)
- else:
- print("Error creating VIM collector")
- # Push to Prometheus
- push_to_gateway(
- gateway=PROMETHEUS_PUSHGW,
- job=f"{PROMETHEUS_JOB_PREFIX}{vim_id}",
- registry=registry,
- )
- return
-
- get_vim_status_and_send_to_prometheus(vim_id)
-
- return dag
-
-
-vim_list = get_all_vim()
-for index, vim in enumerate(vim_list):
- vim_type = vim["vim_type"]
- if vim_type in SUPPORTED_VIM_TYPES:
- vim_id = vim["_id"]
- vim_name = vim["name"]
- dag_description = f"Dag for VIM {vim_name} status"
- dag_id = f"vim_status_{vim_id}"
- print(f"Creating DAG {dag_id}")
- globals()[dag_id] = create_dag(
- dag_id=dag_id,
- dag_number=index,
- dag_description=dag_description,
- vim_id=vim_id,
- )
- else:
- print(f"VIM type '{vim_type}' not supported for monitoring VIM status")
+++ /dev/null
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################
-from datetime import datetime, timedelta
-
-from airflow import DAG
-from airflow.decorators import task
-from osm_mon.core.common_db import CommonDbClient
-from osm_mon.core.config import Config
-from osm_mon.vim_connectors.azure import AzureCollector
-from osm_mon.vim_connectors.gcp import GcpCollector
-from osm_mon.vim_connectors.openstack import OpenStackCollector
-from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
-
-
-SUPPORTED_VIM_TYPES = ["openstack", "vio", "gcp", "azure"]
-PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091"
-PROMETHEUS_JOB_PREFIX = "airflow_osm_vm_status_"
-PROMETHEUS_METRIC = "vm_status"
-PROMETHEUS_METRIC_DESCRIPTION = "VM Status from VIM"
-SCHEDULE_INTERVAL = 1
-
-
-def get_all_vim():
- """Get VIMs from MongoDB"""
- print("Getting VIM list")
-
- cfg = Config()
- print(cfg.conf)
- common_db = CommonDbClient(cfg)
- vim_accounts = common_db.get_vim_accounts()
- vim_list = []
- for vim in vim_accounts:
- print(f'Read VIM {vim["_id"]} ({vim["name"]})')
- vim_list.append(
- {"_id": vim["_id"], "name": vim["name"], "vim_type": vim["vim_type"]}
- )
-
- print(vim_list)
- print("Getting VIM list OK")
- return vim_list
-
-
-def create_dag(dag_id, dag_number, dag_description, vim_id):
- dag = DAG(
- dag_id,
- catchup=False,
- default_args={
- "depends_on_past": False,
- "retries": 1,
- # "retry_delay": timedelta(minutes=1),
- "retry_delay": timedelta(seconds=10),
- },
- description=dag_description,
- is_paused_upon_creation=False,
- # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL),
- schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *",
- start_date=datetime(2022, 1, 1),
- tags=["osm", "vim"],
- )
-
- with dag:
-
- def get_vim_collector(vim_account):
- """Return a VIM collector for the vim_account"""
- vim_type = vim_account["vim_type"]
- if "config" in vim_account and "vim_type" in vim_account["config"]:
- vim_type = vim_account["config"]["vim_type"].lower()
- if vim_type == "vio" and "vrops_site" not in vim_account["config"]:
- vim_type = "openstack"
- if vim_type == "openstack":
- return OpenStackCollector(vim_account)
- if vim_type == "gcp":
- return GcpCollector(vim_account)
- if vim_type == "azure":
- return AzureCollector(vim_account)
- print(f"VIM type '{vim_type}' not supported")
- return None
-
- def get_all_vm_status(vim_account):
- """Get VM status from the VIM"""
- collector = get_vim_collector(vim_account)
- if collector:
- status = collector.is_vim_ok()
- print(f"VIM status: {status}")
- vm_status_list = collector.collect_servers_status()
- return vm_status_list
- else:
- return None
-
- @task(task_id="get_all_vm_status_and_send_to_prometheus")
- def get_all_vm_status_and_send_to_prometheus(vim_id: str):
- """Authenticate against VIM, collect servers status and send to prometheus"""
-
- # Get VIM account info from MongoDB
- print(f"Reading VIM info, id: {vim_id}")
- cfg = Config()
- common_db = CommonDbClient(cfg)
- vim_account = common_db.get_vim_account(vim_account_id=vim_id)
- print(vim_account)
-
- # Define Prometheus Metric for NS topology
- registry = CollectorRegistry()
- metric = Gauge(
- PROMETHEUS_METRIC,
- PROMETHEUS_METRIC_DESCRIPTION,
- labelnames=[
- "vm_id",
- "vim_id",
- ],
- registry=registry,
- )
-
- # Get status of all VM from VIM
- all_vm_status = get_all_vm_status(vim_account)
- print(f"Got {len(all_vm_status)} VMs with their status:")
- if all_vm_status:
- for vm in all_vm_status:
- vm_id = vm["id"]
- vm_status = vm["status"]
- vm_name = vm.get("name", "")
- print(f" {vm_name} ({vm_id}) {vm_status}")
- metric.labels(vm_id, vim_id).set(vm_status)
- # Push to Prometheus only if there are VM
- push_to_gateway(
- gateway=PROMETHEUS_PUSHGW,
- job=f"{PROMETHEUS_JOB_PREFIX}{vim_id}",
- registry=registry,
- )
- return
-
- get_all_vm_status_and_send_to_prometheus(vim_id)
-
- return dag
-
-
-vim_list = get_all_vim()
-for index, vim in enumerate(vim_list):
- vim_type = vim["vim_type"]
- if vim_type in SUPPORTED_VIM_TYPES:
- vim_id = vim["_id"]
- vim_name = vim["name"]
- dag_description = f"Dag for vim {vim_name}"
- dag_id = f"vm_status_vim_{vim_id}"
- print(f"Creating DAG {dag_id}")
- globals()[dag_id] = create_dag(
- dag_id=dag_id,
- dag_number=index,
- dag_description=dag_description,
- vim_id=vim_id,
- )
- else:
- print(f"VIM type '{vim_type}' not supported for collecting VM status")
+++ /dev/null
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################
-from datetime import datetime, timedelta
-
-from airflow.decorators import dag, task
-from osm_mon.core.common_db import CommonDbClient
-from osm_mon.core.config import Config
-from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
-
-
-PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091"
-PROMETHEUS_JOB = "airflow_osm_ns_topology"
-PROMETHEUS_METRIC = "ns_topology"
-PROMETHEUS_METRIC_DESCRIPTION = "Network services topology"
-SCHEDULE_INTERVAL = 2
-
-
-@dag(
- catchup=False,
- default_args={
- "depends_on_past": False,
- "retries": 1,
- "retry_delay": timedelta(seconds=10),
- },
- description="NS topology",
- is_paused_upon_creation=False,
- # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL),
- schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *",
- start_date=datetime(2022, 1, 1),
- tags=["osm", "topology"],
-)
-def ns_topology():
- @task(task_id="get_topology")
- def get_topology():
- """
- Get NS topology from MongoDB and exports it as a metric
- to Prometheus
- """
-
- # Define Prometheus Metric for NS topology
- registry = CollectorRegistry()
- metric = Gauge(
- PROMETHEUS_METRIC,
- PROMETHEUS_METRIC_DESCRIPTION,
- labelnames=[
- "ns_id",
- "project_id",
- "vnf_id",
- "vdu_id",
- "vm_id",
- "vim_id",
- "vdu_name",
- "vnf_member_index",
- ],
- registry=registry,
- )
-
- # Getting VNFR list from MongoDB
- print("Getting VNFR list from MongoDB")
- cfg = Config()
- print(cfg.conf)
- common_db = CommonDbClient(cfg)
- vnfr_list = common_db.get_vnfrs()
-
- # Only send topology if ns state is one of the nsAllowedStatesSet
- nsAllowedStatesSet = {"INSTANTIATED"}
-
- # For loop to get NS topology.
- # For each VDU, a metric sample is created with the appropriate labels
- for vnfr in vnfr_list:
- # Label ns_id
- vnf_id = vnfr["_id"]
- # Label ns_id
- ns_id = vnfr["nsr-id-ref"]
- # Label vnfd_id
- vnfd_id = vnfr["vnfd-ref"]
- # Label project_id
- project_list = vnfr.get("_admin", {}).get("projects_read", [])
- project_id = "None"
- if project_list:
- project_id = project_list[0]
- # TODO: use logger with loglevels instead of print
- # Other info
- ns_state = vnfr["_admin"]["nsState"]
- vnf_membex_index = vnfr["member-vnf-index-ref"]
- print(
- f"Read VNFR: id: {vnf_id}, ns_id: {ns_id}, ",
- f"state: {ns_state}, vnfd_id: {vnfd_id}, ",
- f"vnf_membex_index: {vnf_membex_index}, ",
- f"project_id: {project_id}",
- )
- # Only send topology if ns State is one of the nsAllowedStatesSet
- if ns_state not in nsAllowedStatesSet:
- continue
-
- print("VDU list:")
- for vdu in vnfr.get("vdur", []):
- # Label vdu_id
- vdu_id = vdu["_id"]
- # Label vim_id
- vim_info = vdu.get("vim_info")
- if not vim_info:
- print("Error: vim_info not available in vdur")
- continue
- if len(vim_info) != 1:
- print("Error: more than one vim_info in vdur")
- continue
- vim_id = next(iter(vim_info))[4:]
- # Label vm_id
- vm_id = vdu["vim-id"]
- # Other VDU info
- vdu_name = vdu.get("name", "UNKNOWN")
- print(
- f" id: {vdu_id}, name: {vdu_name}, "
- f"vim_id: {vim_id}, vm_id: {vm_id}"
- )
- print(
- f"METRIC SAMPLE: ns_id: {ns_id}, ",
- f"project_id: {project_id}, vnf_id: {vnf_id}, ",
- f"vdu_id: {vdu_id}, vm_id: {vm_id}, vim_id: {vim_id}",
- )
- metric.labels(
- ns_id,
- project_id,
- vnf_id,
- vdu_id,
- vm_id,
- vim_id,
- vdu_name,
- vnf_membex_index,
- ).set(1)
-
- # print("Push to gateway")
- push_to_gateway(
- gateway=PROMETHEUS_PUSHGW, job=PROMETHEUS_JOB, registry=registry
- )
- return
-
- get_topology()
-
-
-dag = ns_topology()
+++ /dev/null
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################
-from osm_common import dbmemory, dbmongo
-from osm_mon.core.config import Config
-
-
-class CommonDbClient:
- def __init__(self, config: Config):
- if config.get("database", "driver") == "mongo":
- self.common_db = dbmongo.DbMongo()
- elif config.get("database", "driver") == "memory":
- self.common_db = dbmemory.DbMemory()
- else:
- raise Exception(
- "Unknown database driver {}".format(config.get("section", "driver"))
- )
- self.common_db.db_connect(config.get("database"))
-
- def get_vnfr(self, nsr_id: str, member_index: int):
- vnfr = self.common_db.get_one(
- "vnfrs", {"nsr-id-ref": nsr_id, "member-vnf-index-ref": str(member_index)}
- )
- return vnfr
-
- def get_vnfrs(self, nsr_id: str = None, vim_account_id: str = None):
- if nsr_id and vim_account_id:
- raise NotImplementedError("Only one filter is currently supported")
- if nsr_id:
- vnfrs = [
- self.get_vnfr(nsr_id, member["member-vnf-index"])
- for member in self.get_nsr(nsr_id)["nsd"]["constituent-vnfd"]
- ]
- elif vim_account_id:
- vnfrs = self.common_db.get_list("vnfrs", {"vim-account-id": vim_account_id})
- else:
- vnfrs = self.common_db.get_list("vnfrs")
- return vnfrs
-
- def get_nsr(self, nsr_id: str):
- nsr = self.common_db.get_one("nsrs", {"id": nsr_id})
- return nsr
-
- def decrypt_vim_password(self, vim_password: str, schema_version: str, vim_id: str):
- return self.common_db.decrypt(vim_password, schema_version, vim_id)
-
- def get_vim_accounts(self):
- return self.common_db.get_list("vim_accounts")
-
- def get_vim_account(self, vim_account_id: str) -> dict:
- vim_account = self.common_db.get_one("vim_accounts", {"_id": vim_account_id})
- vim_account["vim_password"] = self.decrypt_vim_password(
- vim_account["vim_password"], vim_account["schema_version"], vim_account_id
- )
- vim_config_encrypted_dict = {
- "1.1": ("admin_password", "nsx_password", "vcenter_password"),
- "default": (
- "admin_password",
- "nsx_password",
- "vcenter_password",
- "vrops_password",
- ),
- }
- vim_config_encrypted = vim_config_encrypted_dict["default"]
- if vim_account["schema_version"] in vim_config_encrypted_dict.keys():
- vim_config_encrypted = vim_config_encrypted_dict[
- vim_account["schema_version"]
- ]
- if "config" in vim_account:
- for key in vim_account["config"]:
- if key in vim_config_encrypted:
- vim_account["config"][key] = self.decrypt_vim_password(
- vim_account["config"][key],
- vim_account["schema_version"],
- vim_account_id,
- )
- return vim_account
+++ /dev/null
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################
-"""Global configuration managed by environment variables."""
-
-import logging
-import os
-
-import pkg_resources
-import yaml
-
-logger = logging.getLogger(__name__)
-
-
-class Config:
- def __init__(self, config_file: str = ""):
- self.conf = {}
- self._read_config_file(config_file)
- self._read_env()
-
- def _read_config_file(self, config_file):
- if not config_file:
- path = "config.yaml"
- config_file = pkg_resources.resource_filename(__name__, path)
- with open(config_file) as f:
- self.conf = yaml.load(f)
-
- def get(self, section, field=None):
- if not field:
- return self.conf[section]
- return self.conf[section].get(field)
-
- def set(self, section, field, value):
- if section not in self.conf:
- self.conf[section] = {}
- self.conf[section][field] = value
-
- def _read_env(self):
- for env in os.environ:
- if not env.startswith("OSMMON_"):
- continue
- elements = env.lower().split("_")
- if len(elements) < 3:
- logger.warning(
- "Environment variable %s=%s does not comply with required format. Section and/or field missing.",
- env,
- os.getenv(env),
- )
- continue
- section = elements[1]
- field = "_".join(elements[2:])
- value = os.getenv(env)
- if section not in self.conf:
- self.conf[section] = {}
- self.conf[section][field] = value
+++ /dev/null
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################
-
-global:
- loglevel: INFO
- request_timeout: 10
-
-database:
- driver: mongo
- uri: mongodb://mongodb-k8s:27017/?replicaSet=rs0
- name: osm
- commonkey: gj7LmbCexbmII7memwbGRRdfbYuT3nvy
-
+++ /dev/null
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################
-import logging
-from typing import Dict, List
-
-from azure.identity import ClientSecretCredential
-from azure.mgmt.compute import ComputeManagementClient
-from azure.profiles import ProfileDefinition
-from osm_mon.vim_connectors.base_vim import VIMConnector
-
-
-log = logging.getLogger(__name__)
-
-
-class AzureCollector(VIMConnector):
-
- # Translate azure provisioning state to OSM provision state.
- # The first three ones are the transitional status once a user initiated
- # action has been requested. Once the operation is complete, it will
- # transition into the states Succeeded or Failed
- # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/states-lifecycle
- provision_state2osm = {
- "Creating": "BUILD",
- "Updating": "BUILD",
- "Deleting": "INACTIVE",
- "Succeeded": "ACTIVE",
- "Failed": "ERROR",
- }
-
- # Translate azure power state to OSM provision state
- power_state2osm = {
- "starting": "INACTIVE",
- "running": "ACTIVE",
- "stopping": "INACTIVE",
- "stopped": "INACTIVE",
- "unknown": "OTHER",
- "deallocated": "BUILD",
- "deallocating": "BUILD",
- }
-
- AZURE_COMPUTE_MGMT_CLIENT_API_VERSION = "2021-03-01"
- AZURE_COMPUTE_MGMT_PROFILE_TAG = "azure.mgmt.compute.ComputeManagementClient"
- AZURE_COMPUTE_MGMT_PROFILE = ProfileDefinition(
- {
- AZURE_COMPUTE_MGMT_PROFILE_TAG: {
- None: AZURE_COMPUTE_MGMT_CLIENT_API_VERSION,
- "availability_sets": "2020-12-01",
- "dedicated_host_groups": "2020-12-01",
- "dedicated_hosts": "2020-12-01",
- "disk_accesses": "2020-12-01",
- "disk_encryption_sets": "2020-12-01",
- "disk_restore_point": "2020-12-01",
- "disks": "2020-12-01",
- "galleries": "2020-09-30",
- "gallery_application_versions": "2020-09-30",
- "gallery_applications": "2020-09-30",
- "gallery_image_versions": "2020-09-30",
- "gallery_images": "2020-09-30",
- "gallery_sharing_profile": "2020-09-30",
- "images": "2020-12-01",
- "log_analytics": "2020-12-01",
- "operations": "2020-12-01",
- "proximity_placement_groups": "2020-12-01",
- "resource_skus": "2019-04-01",
- "shared_galleries": "2020-09-30",
- "shared_gallery_image_versions": "2020-09-30",
- "shared_gallery_images": "2020-09-30",
- "snapshots": "2020-12-01",
- "ssh_public_keys": "2020-12-01",
- "usage": "2020-12-01",
- "virtual_machine_extension_images": "2020-12-01",
- "virtual_machine_extensions": "2020-12-01",
- "virtual_machine_images": "2020-12-01",
- "virtual_machine_images_edge_zone": "2020-12-01",
- "virtual_machine_run_commands": "2020-12-01",
- "virtual_machine_scale_set_extensions": "2020-12-01",
- "virtual_machine_scale_set_rolling_upgrades": "2020-12-01",
- "virtual_machine_scale_set_vm_extensions": "2020-12-01",
- "virtual_machine_scale_set_vm_run_commands": "2020-12-01",
- "virtual_machine_scale_set_vms": "2020-12-01",
- "virtual_machine_scale_sets": "2020-12-01",
- "virtual_machine_sizes": "2020-12-01",
- "virtual_machines": "2020-12-01",
- }
- },
- AZURE_COMPUTE_MGMT_PROFILE_TAG + " osm",
- )
-
- def __init__(self, vim_account: Dict):
- self.vim_account = vim_account
- self.reload_client = True
- logger = logging.getLogger("azure")
- logger.setLevel(logging.ERROR)
- # Store config to create azure subscription later
- self._config = {
- "user": vim_account["vim_user"],
- "passwd": vim_account["vim_password"],
- "tenant": vim_account["vim_tenant_name"],
- }
-
- # SUBSCRIPTION
- config = vim_account["config"]
- if "subscription_id" in config:
- self._config["subscription_id"] = config.get("subscription_id")
- log.info("Subscription: %s", self._config["subscription_id"])
- else:
- log.error("Subscription not specified")
- return
-
- # RESOURCE_GROUP
- if "resource_group" in config:
- self.resource_group = config.get("resource_group")
- else:
- log.error("Azure resource_group is not specified at config")
- return
-
- def _reload_connection(self):
- if self.reload_client:
- log.debug("reloading azure client")
- try:
- self.credentials = ClientSecretCredential(
- client_id=self._config["user"],
- client_secret=self._config["passwd"],
- tenant_id=self._config["tenant"],
- )
- self.conn_compute = ComputeManagementClient(
- self.credentials,
- self._config["subscription_id"],
- profile=self.AZURE_COMPUTE_MGMT_PROFILE,
- )
- # Set to client created
- self.reload_client = False
- except Exception as e:
- log.error(e)
-
- def collect_servers_status(self) -> List[Dict]:
- servers = []
- log.debug("collect_servers_status")
- self._reload_connection()
- try:
- for vm in self.conn_compute.virtual_machines.list(self.resource_group):
- id = vm.id
- array = id.split("/")
- name = array[-1]
- status = self.provision_state2osm.get(vm.provisioning_state, "OTHER")
- if vm.provisioning_state == "Succeeded":
- # check if machine is running or stopped
- instance_view = self.conn_compute.virtual_machines.instance_view(
- self.resource_group, name
- )
- for status in instance_view.statuses:
- splitted_status = status.code.split("/")
- if (
- len(splitted_status) == 2
- and splitted_status[0] == "PowerState"
- ):
- status = self.power_state2osm.get(
- splitted_status[1], "OTHER"
- )
- # log.info(f'id: {id}, name: {name}, status: {status}')
- vm = {
- "id": id,
- "name": name,
- "status": (1 if (status == "ACTIVE") else 0),
- }
- servers.append(vm)
- except Exception as e:
- log.error(e)
- return servers
-
- def is_vim_ok(self) -> bool:
- status = False
- self.reload_client = True
- try:
- self._reload_connection()
- status = True
- except Exception as e:
- log.error(e)
- return status
+++ /dev/null
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################
-from typing import Dict, List
-
-
-class VIMConnector:
- def __init__(self, vim_account: Dict):
- pass
-
- # def collect_servers_status(self) -> List[Metric]:
- def collect_servers_status(self) -> List:
- pass
-
- def is_vim_ok(self) -> bool:
- pass
+++ /dev/null
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################
-# pylint: disable=E1101
-
-import logging
-from typing import Dict, List
-
-from google.oauth2 import service_account
-import googleapiclient.discovery
-from osm_mon.vim_connectors.base_vim import VIMConnector
-
-log = logging.getLogger(__name__)
-
-
-class GcpCollector(VIMConnector):
- def __init__(self, vim_account: Dict):
- self.vim_account = vim_account
- self.project = vim_account["vim_tenant_name"] or vim_account["vim_tenant_id"]
-
- # REGION - Google Cloud considers regions and zones. A specific region
- # can have more than one zone (for instance: region us-west1 with the
- # zones us-west1-a, us-west1-b and us-west1-c). So the region name
- # specified in the config will be considered as a specific zone for GC
- # and the region will be calculated from that without the preffix.
- if "config" in vim_account:
- config = vim_account["config"]
- if "region_name" in config:
- self.zone = config.get("region_name")
- self.region = self.zone.rsplit("-", 1)[0]
- else:
- log.error("Google Cloud region_name not specified in config")
- else:
- log.error("config is not specified in VIM")
-
- # Credentials
- scopes = ["https://www.googleapis.com/auth/cloud-platform"]
- self.credentials = None
- if "credentials" in config:
- log.debug("Setting credentials")
- # Settings Google Cloud credentials dict
- creds_body = config["credentials"]
- creds = service_account.Credentials.from_service_account_info(creds_body)
- if "sa_file" in config:
- creds = service_account.Credentials.from_service_account_file(
- config.get("sa_file"), scopes=scopes
- )
- log.debug("Credentials: %s", creds)
- # Construct a Resource for interacting with an API.
- self.credentials = creds
- try:
- self.conn_compute = googleapiclient.discovery.build(
- "compute", "v1", credentials=creds
- )
- except Exception as e:
- log.error(e)
- else:
- log.error("It is not possible to init GCP with no credentials")
-
- def collect_servers_status(self) -> List[Dict]:
- servers = []
- try:
- response = (
- self.conn_compute.instances()
- .list(project=self.project, zone=self.zone)
- .execute()
- )
- if "items" in response:
- log.info(response["items"])
- for server in response["items"]:
- vm = {
- "id": server["id"],
- "name": server["name"],
- "status": (1 if (server["status"] == "RUNNING") else 0),
- }
- servers.append(vm)
- except Exception as e:
- log.error(e)
- return servers
-
- def is_vim_ok(self) -> bool:
- status = False
- try:
- self.conn_compute.zones().get(
- project=self.project, zone=self.zone
- ).execute()
- status = True
- except Exception as e:
- log.error(e)
- return status
+++ /dev/null
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################
-import logging
-from typing import Dict, List
-
-from keystoneauth1 import session
-from keystoneauth1.identity import v3
-from novaclient import client as nova_client
-from osm_mon.vim_connectors.base_vim import VIMConnector
-
-log = logging.getLogger(__name__)
-
-
-class CertificateNotCreated(Exception):
- pass
-
-
-class OpenStackCollector(VIMConnector):
- def __init__(self, vim_account: Dict):
- log.info("__init__")
- self.vim_account = vim_account
- self.vim_session = None
- self.vim_session = self._get_session(vim_account)
- self.nova = self._build_nova_client()
-
- def _get_session(self, creds: Dict):
- verify_ssl = True
- project_domain_name = "Default"
- user_domain_name = "Default"
- try:
- if "config" in creds:
- vim_config = creds["config"]
- if "insecure" in vim_config and vim_config["insecure"]:
- verify_ssl = False
- if "ca_cert" in vim_config:
- verify_ssl = vim_config["ca_cert"]
- elif "ca_cert_content" in vim_config:
- # vim_config = self._create_file_cert(vim_config, creds["_id"])
- verify_ssl = vim_config["ca_cert"]
- if "project_domain_name" in vim_config:
- project_domain_name = vim_config["project_domain_name"]
- if "user_domain_name" in vim_config:
- user_domain_name = vim_config["user_domain_name"]
- auth = v3.Password(
- auth_url=creds["vim_url"],
- username=creds["vim_user"],
- password=creds["vim_password"],
- project_name=creds["vim_tenant_name"],
- project_domain_name=project_domain_name,
- user_domain_name=user_domain_name,
- )
- return session.Session(auth=auth, verify=verify_ssl, timeout=10)
- except CertificateNotCreated as e:
- log.error(e)
-
- def _build_nova_client(self) -> nova_client.Client:
- return nova_client.Client("2", session=self.vim_session, timeout=10)
-
- def collect_servers_status(self) -> List[Dict]:
- log.info("collect_servers_status")
- servers = []
- for server in self.nova.servers.list(detailed=True):
- vm = {
- "id": server.id,
- "name": server.name,
- "status": (0 if (server.status == "ERROR") else 1),
- }
- servers.append(vm)
- return servers
-
- def is_vim_ok(self) -> bool:
- try:
- self.nova.servers.list()
- return True
- except Exception as e:
- log.warning("VIM status is not OK: %s" % e)
- return False
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+from datetime import datetime, timedelta
+
+from airflow import DAG
+from airflow.decorators import task
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+from osm_mon.vim_connectors.azure import AzureCollector
+from osm_mon.vim_connectors.gcp import GcpCollector
+from osm_mon.vim_connectors.openstack import OpenStackCollector
+from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
+
+
+SUPPORTED_VIM_TYPES = ["openstack", "vio", "gcp", "azure"]
+PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091"
+PROMETHEUS_JOB_PREFIX = "airflow_osm_vim_status_"
+PROMETHEUS_METRIC = "vim_status"
+PROMETHEUS_METRIC_DESCRIPTION = "VIM status"
+SCHEDULE_INTERVAL = 1
+
+
+def get_all_vim():
+ """Get VIMs from MongoDB"""
+ print("Getting VIM list")
+
+ cfg = Config()
+ print(cfg.conf)
+ common_db = CommonDbClient(cfg)
+ vim_accounts = common_db.get_vim_accounts()
+ vim_list = []
+ for vim in vim_accounts:
+ print(f'Read VIM {vim["_id"]} ({vim["name"]})')
+ vim_list.append(
+ {"_id": vim["_id"], "name": vim["name"], "vim_type": vim["vim_type"]}
+ )
+
+ print(vim_list)
+ print("Getting VIM list OK")
+ return vim_list
+
+
+def create_dag(dag_id, dag_number, dag_description, vim_id):
+ dag = DAG(
+ dag_id,
+ catchup=False,
+ default_args={
+ "depends_on_past": False,
+ "retries": 1,
+ # "retry_delay": timedelta(minutes=1),
+ "retry_delay": timedelta(seconds=10),
+ },
+ description=dag_description,
+ is_paused_upon_creation=False,
+ # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL),
+ schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *",
+ start_date=datetime(2022, 1, 1),
+ tags=["osm", "vim"],
+ )
+
+ with dag:
+
+ def get_vim_collector(vim_account):
+ """Return a VIM collector for the vim_account"""
+ vim_type = vim_account["vim_type"]
+ if "config" in vim_account and "vim_type" in vim_account["config"]:
+ vim_type = vim_account["config"]["vim_type"].lower()
+ if vim_type == "vio" and "vrops_site" not in vim_account["config"]:
+ vim_type = "openstack"
+ if vim_type == "openstack":
+ return OpenStackCollector(vim_account)
+ if vim_type == "gcp":
+ return GcpCollector(vim_account)
+ if vim_type == "azure":
+ return AzureCollector(vim_account)
+ print(f"VIM type '{vim_type}' not supported")
+ return None
+
+ @task(task_id="get_vim_status_and_send_to_prometheus")
+ def get_vim_status_and_send_to_prometheus(vim_id: str):
+ """Authenticate against VIM and check status"""
+
+ # Get VIM account info from MongoDB
+ print(f"Reading VIM info, id: {vim_id}")
+ cfg = Config()
+ common_db = CommonDbClient(cfg)
+ vim_account = common_db.get_vim_account(vim_account_id=vim_id)
+ print(vim_account)
+
+ # Define Prometheus Metric for NS topology
+ registry = CollectorRegistry()
+ metric = Gauge(
+ PROMETHEUS_METRIC,
+ PROMETHEUS_METRIC_DESCRIPTION,
+ labelnames=[
+ "vim_id",
+ ],
+ registry=registry,
+ )
+ metric.labels(vim_id).set(0)
+
+ # Get status of VIM
+ collector = get_vim_collector(vim_account)
+ if collector:
+ status = collector.is_vim_ok()
+ print(f"VIM status: {status}")
+ metric.labels(vim_id).set(1)
+ else:
+ print("Error creating VIM collector")
+ # Push to Prometheus
+ push_to_gateway(
+ gateway=PROMETHEUS_PUSHGW,
+ job=f"{PROMETHEUS_JOB_PREFIX}{vim_id}",
+ registry=registry,
+ )
+ return
+
+ get_vim_status_and_send_to_prometheus(vim_id)
+
+ return dag
+
+
+vim_list = get_all_vim()
+for index, vim in enumerate(vim_list):
+ vim_type = vim["vim_type"]
+ if vim_type in SUPPORTED_VIM_TYPES:
+ vim_id = vim["_id"]
+ vim_name = vim["name"]
+ dag_description = f"Dag for VIM {vim_name} status"
+ dag_id = f"vim_status_{vim_id}"
+ print(f"Creating DAG {dag_id}")
+ globals()[dag_id] = create_dag(
+ dag_id=dag_id,
+ dag_number=index,
+ dag_description=dag_description,
+ vim_id=vim_id,
+ )
+ else:
+ print(f"VIM type '{vim_type}' not supported for monitoring VIM status")
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+from datetime import datetime, timedelta
+
+from airflow import DAG
+from airflow.decorators import task
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+from osm_mon.vim_connectors.azure import AzureCollector
+from osm_mon.vim_connectors.gcp import GcpCollector
+from osm_mon.vim_connectors.openstack import OpenStackCollector
+from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
+
+
+SUPPORTED_VIM_TYPES = ["openstack", "vio", "gcp", "azure"]
+PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091"
+PROMETHEUS_JOB_PREFIX = "airflow_osm_vm_status_"
+PROMETHEUS_METRIC = "vm_status"
+PROMETHEUS_METRIC_DESCRIPTION = "VM Status from VIM"
+SCHEDULE_INTERVAL = 1
+
+
+def get_all_vim():
+ """Get VIMs from MongoDB"""
+ print("Getting VIM list")
+
+ cfg = Config()
+ print(cfg.conf)
+ common_db = CommonDbClient(cfg)
+ vim_accounts = common_db.get_vim_accounts()
+ vim_list = []
+ for vim in vim_accounts:
+ print(f'Read VIM {vim["_id"]} ({vim["name"]})')
+ vim_list.append(
+ {"_id": vim["_id"], "name": vim["name"], "vim_type": vim["vim_type"]}
+ )
+
+ print(vim_list)
+ print("Getting VIM list OK")
+ return vim_list
+
+
+def create_dag(dag_id, dag_number, dag_description, vim_id):
+ dag = DAG(
+ dag_id,
+ catchup=False,
+ default_args={
+ "depends_on_past": False,
+ "retries": 1,
+ # "retry_delay": timedelta(minutes=1),
+ "retry_delay": timedelta(seconds=10),
+ },
+ description=dag_description,
+ is_paused_upon_creation=False,
+ # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL),
+ schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *",
+ start_date=datetime(2022, 1, 1),
+ tags=["osm", "vim"],
+ )
+
+ with dag:
+
+ def get_vim_collector(vim_account):
+ """Return a VIM collector for the vim_account"""
+ vim_type = vim_account["vim_type"]
+ if "config" in vim_account and "vim_type" in vim_account["config"]:
+ vim_type = vim_account["config"]["vim_type"].lower()
+ if vim_type == "vio" and "vrops_site" not in vim_account["config"]:
+ vim_type = "openstack"
+ if vim_type == "openstack":
+ return OpenStackCollector(vim_account)
+ if vim_type == "gcp":
+ return GcpCollector(vim_account)
+ if vim_type == "azure":
+ return AzureCollector(vim_account)
+ print(f"VIM type '{vim_type}' not supported")
+ return None
+
+ def get_all_vm_status(vim_account):
+ """Get VM status from the VIM"""
+ collector = get_vim_collector(vim_account)
+ if collector:
+ status = collector.is_vim_ok()
+ print(f"VIM status: {status}")
+ vm_status_list = collector.collect_servers_status()
+ return vm_status_list
+ else:
+ return None
+
+ @task(task_id="get_all_vm_status_and_send_to_prometheus")
+ def get_all_vm_status_and_send_to_prometheus(vim_id: str):
+ """Authenticate against VIM, collect servers status and send to prometheus"""
+
+ # Get VIM account info from MongoDB
+ print(f"Reading VIM info, id: {vim_id}")
+ cfg = Config()
+ common_db = CommonDbClient(cfg)
+ vim_account = common_db.get_vim_account(vim_account_id=vim_id)
+ print(vim_account)
+
+ # Define Prometheus Metric for NS topology
+ registry = CollectorRegistry()
+ metric = Gauge(
+ PROMETHEUS_METRIC,
+ PROMETHEUS_METRIC_DESCRIPTION,
+ labelnames=[
+ "vm_id",
+ "vim_id",
+ ],
+ registry=registry,
+ )
+
+ # Get status of all VM from VIM
+ all_vm_status = get_all_vm_status(vim_account)
+ print(f"Got {len(all_vm_status)} VMs with their status:")
+ if all_vm_status:
+ for vm in all_vm_status:
+ vm_id = vm["id"]
+ vm_status = vm["status"]
+ vm_name = vm.get("name", "")
+ print(f" {vm_name} ({vm_id}) {vm_status}")
+ metric.labels(vm_id, vim_id).set(vm_status)
+ # Push to Prometheus only if there are VM
+ push_to_gateway(
+ gateway=PROMETHEUS_PUSHGW,
+ job=f"{PROMETHEUS_JOB_PREFIX}{vim_id}",
+ registry=registry,
+ )
+ return
+
+ get_all_vm_status_and_send_to_prometheus(vim_id)
+
+ return dag
+
+
+vim_list = get_all_vim()
+for index, vim in enumerate(vim_list):
+ vim_type = vim["vim_type"]
+ if vim_type in SUPPORTED_VIM_TYPES:
+ vim_id = vim["_id"]
+ vim_name = vim["name"]
+ dag_description = f"Dag for vim {vim_name}"
+ dag_id = f"vm_status_vim_{vim_id}"
+ print(f"Creating DAG {dag_id}")
+ globals()[dag_id] = create_dag(
+ dag_id=dag_id,
+ dag_number=index,
+ dag_description=dag_description,
+ vim_id=vim_id,
+ )
+ else:
+ print(f"VIM type '{vim_type}' not supported for collecting VM status")
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+from datetime import datetime, timedelta
+
+from airflow.decorators import dag, task
+from osm_mon.core.common_db import CommonDbClient
+from osm_mon.core.config import Config
+from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
+
+
+PROMETHEUS_PUSHGW = "pushgateway-prometheus-pushgateway:9091"
+PROMETHEUS_JOB = "airflow_osm_ns_topology"
+PROMETHEUS_METRIC = "ns_topology"
+PROMETHEUS_METRIC_DESCRIPTION = "Network services topology"
+SCHEDULE_INTERVAL = 2
+
+
+@dag(
+ catchup=False,
+ default_args={
+ "depends_on_past": False,
+ "retries": 1,
+ "retry_delay": timedelta(seconds=10),
+ },
+ description="NS topology",
+ is_paused_upon_creation=False,
+ # schedule_interval=timedelta(minutes=SCHEDULE_INTERVAL),
+ schedule_interval=f"*/{SCHEDULE_INTERVAL} * * * *",
+ start_date=datetime(2022, 1, 1),
+ tags=["osm", "topology"],
+)
+def ns_topology():
+ @task(task_id="get_topology")
+ def get_topology():
+ """
+ Get NS topology from MongoDB and exports it as a metric
+ to Prometheus
+ """
+
+ # Define Prometheus Metric for NS topology
+ registry = CollectorRegistry()
+ metric = Gauge(
+ PROMETHEUS_METRIC,
+ PROMETHEUS_METRIC_DESCRIPTION,
+ labelnames=[
+ "ns_id",
+ "project_id",
+ "vnf_id",
+ "vdu_id",
+ "vm_id",
+ "vim_id",
+ "vdu_name",
+ "vnf_member_index",
+ ],
+ registry=registry,
+ )
+
+ # Getting VNFR list from MongoDB
+ print("Getting VNFR list from MongoDB")
+ cfg = Config()
+ print(cfg.conf)
+ common_db = CommonDbClient(cfg)
+ vnfr_list = common_db.get_vnfrs()
+
+ # Only send topology if ns state is one of the nsAllowedStatesSet
+ nsAllowedStatesSet = {"INSTANTIATED"}
+
+ # For loop to get NS topology.
+ # For each VDU, a metric sample is created with the appropriate labels
+ for vnfr in vnfr_list:
+ # Label ns_id
+ vnf_id = vnfr["_id"]
+ # Label ns_id
+ ns_id = vnfr["nsr-id-ref"]
+ # Label vnfd_id
+ vnfd_id = vnfr["vnfd-ref"]
+ # Label project_id
+ project_list = vnfr.get("_admin", {}).get("projects_read", [])
+ project_id = "None"
+ if project_list:
+ project_id = project_list[0]
+ # TODO: use logger with loglevels instead of print
+ # Other info
+ ns_state = vnfr["_admin"]["nsState"]
+ vnf_membex_index = vnfr["member-vnf-index-ref"]
+ print(
+ f"Read VNFR: id: {vnf_id}, ns_id: {ns_id}, ",
+ f"state: {ns_state}, vnfd_id: {vnfd_id}, ",
+ f"vnf_membex_index: {vnf_membex_index}, ",
+ f"project_id: {project_id}",
+ )
+ # Only send topology if ns State is one of the nsAllowedStatesSet
+ if ns_state not in nsAllowedStatesSet:
+ continue
+
+ print("VDU list:")
+ for vdu in vnfr.get("vdur", []):
+ # Label vdu_id
+ vdu_id = vdu["_id"]
+ # Label vim_id
+ vim_info = vdu.get("vim_info")
+ if not vim_info:
+ print("Error: vim_info not available in vdur")
+ continue
+ if len(vim_info) != 1:
+ print("Error: more than one vim_info in vdur")
+ continue
+ vim_id = next(iter(vim_info))[4:]
+ # Label vm_id
+ vm_id = vdu["vim-id"]
+ # Other VDU info
+ vdu_name = vdu.get("name", "UNKNOWN")
+ print(
+ f" id: {vdu_id}, name: {vdu_name}, "
+ f"vim_id: {vim_id}, vm_id: {vm_id}"
+ )
+ print(
+ f"METRIC SAMPLE: ns_id: {ns_id}, ",
+ f"project_id: {project_id}, vnf_id: {vnf_id}, ",
+ f"vdu_id: {vdu_id}, vm_id: {vm_id}, vim_id: {vim_id}",
+ )
+ metric.labels(
+ ns_id,
+ project_id,
+ vnf_id,
+ vdu_id,
+ vm_id,
+ vim_id,
+ vdu_name,
+ vnf_membex_index,
+ ).set(1)
+
+ # print("Push to gateway")
+ push_to_gateway(
+ gateway=PROMETHEUS_PUSHGW, job=PROMETHEUS_JOB, registry=registry
+ )
+ return
+
+ get_topology()
+
+
+dag = ns_topology()
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+from osm_common import dbmemory, dbmongo
+from osm_mon.core.config import Config
+
+
+class CommonDbClient:
+ def __init__(self, config: Config):
+ if config.get("database", "driver") == "mongo":
+ self.common_db = dbmongo.DbMongo()
+ elif config.get("database", "driver") == "memory":
+ self.common_db = dbmemory.DbMemory()
+ else:
+ raise Exception(
+ "Unknown database driver {}".format(config.get("section", "driver"))
+ )
+ self.common_db.db_connect(config.get("database"))
+
+ def get_vnfr(self, nsr_id: str, member_index: int):
+ vnfr = self.common_db.get_one(
+ "vnfrs", {"nsr-id-ref": nsr_id, "member-vnf-index-ref": str(member_index)}
+ )
+ return vnfr
+
+ def get_vnfrs(self, nsr_id: str = None, vim_account_id: str = None):
+ if nsr_id and vim_account_id:
+ raise NotImplementedError("Only one filter is currently supported")
+ if nsr_id:
+ vnfrs = [
+ self.get_vnfr(nsr_id, member["member-vnf-index"])
+ for member in self.get_nsr(nsr_id)["nsd"]["constituent-vnfd"]
+ ]
+ elif vim_account_id:
+ vnfrs = self.common_db.get_list("vnfrs", {"vim-account-id": vim_account_id})
+ else:
+ vnfrs = self.common_db.get_list("vnfrs")
+ return vnfrs
+
+ def get_nsr(self, nsr_id: str):
+ nsr = self.common_db.get_one("nsrs", {"id": nsr_id})
+ return nsr
+
+ def decrypt_vim_password(self, vim_password: str, schema_version: str, vim_id: str):
+ return self.common_db.decrypt(vim_password, schema_version, vim_id)
+
+ def get_vim_accounts(self):
+ return self.common_db.get_list("vim_accounts")
+
+ def get_vim_account(self, vim_account_id: str) -> dict:
+ vim_account = self.common_db.get_one("vim_accounts", {"_id": vim_account_id})
+ vim_account["vim_password"] = self.decrypt_vim_password(
+ vim_account["vim_password"], vim_account["schema_version"], vim_account_id
+ )
+ vim_config_encrypted_dict = {
+ "1.1": ("admin_password", "nsx_password", "vcenter_password"),
+ "default": (
+ "admin_password",
+ "nsx_password",
+ "vcenter_password",
+ "vrops_password",
+ ),
+ }
+ vim_config_encrypted = vim_config_encrypted_dict["default"]
+ if vim_account["schema_version"] in vim_config_encrypted_dict.keys():
+ vim_config_encrypted = vim_config_encrypted_dict[
+ vim_account["schema_version"]
+ ]
+ if "config" in vim_account:
+ for key in vim_account["config"]:
+ if key in vim_config_encrypted:
+ vim_account["config"][key] = self.decrypt_vim_password(
+ vim_account["config"][key],
+ vim_account["schema_version"],
+ vim_account_id,
+ )
+ return vim_account
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+"""Global configuration managed by environment variables."""
+
+import logging
+import os
+
+import pkg_resources
+import yaml
+
+logger = logging.getLogger(__name__)
+
+
+class Config:
+ def __init__(self, config_file: str = ""):
+ self.conf = {}
+ self._read_config_file(config_file)
+ self._read_env()
+
+ def _read_config_file(self, config_file):
+ if not config_file:
+ path = "config.yaml"
+ config_file = pkg_resources.resource_filename(__name__, path)
+ with open(config_file) as f:
+ self.conf = yaml.load(f)
+
+ def get(self, section, field=None):
+ if not field:
+ return self.conf[section]
+ return self.conf[section].get(field)
+
+ def set(self, section, field, value):
+ if section not in self.conf:
+ self.conf[section] = {}
+ self.conf[section][field] = value
+
+ def _read_env(self):
+ for env in os.environ:
+ if not env.startswith("OSMMON_"):
+ continue
+ elements = env.lower().split("_")
+ if len(elements) < 3:
+ logger.warning(
+ "Environment variable %s=%s does not comply with required format. Section and/or field missing.",
+ env,
+ os.getenv(env),
+ )
+ continue
+ section = elements[1]
+ field = "_".join(elements[2:])
+ value = os.getenv(env)
+ if section not in self.conf:
+ self.conf[section] = {}
+ self.conf[section][field] = value
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+
+global:
+ loglevel: INFO
+ request_timeout: 10
+
+database:
+ driver: mongo
+ uri: mongodb://mongodb-k8s:27017/?replicaSet=rs0
+ name: osm
+ commonkey: gj7LmbCexbmII7memwbGRRdfbYuT3nvy
+
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+import logging
+from typing import Dict, List
+
+from azure.identity import ClientSecretCredential
+from azure.mgmt.compute import ComputeManagementClient
+from azure.profiles import ProfileDefinition
+from osm_mon.vim_connectors.base_vim import VIMConnector
+
+
+log = logging.getLogger(__name__)
+
+
+class AzureCollector(VIMConnector):
+
+ # Translate azure provisioning state to OSM provision state.
+ # The first three ones are the transitional status once a user initiated
+ # action has been requested. Once the operation is complete, it will
+ # transition into the states Succeeded or Failed
+ # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/states-lifecycle
+ provision_state2osm = {
+ "Creating": "BUILD",
+ "Updating": "BUILD",
+ "Deleting": "INACTIVE",
+ "Succeeded": "ACTIVE",
+ "Failed": "ERROR",
+ }
+
+ # Translate azure power state to OSM provision state
+ power_state2osm = {
+ "starting": "INACTIVE",
+ "running": "ACTIVE",
+ "stopping": "INACTIVE",
+ "stopped": "INACTIVE",
+ "unknown": "OTHER",
+ "deallocated": "BUILD",
+ "deallocating": "BUILD",
+ }
+
+ AZURE_COMPUTE_MGMT_CLIENT_API_VERSION = "2021-03-01"
+ AZURE_COMPUTE_MGMT_PROFILE_TAG = "azure.mgmt.compute.ComputeManagementClient"
+ AZURE_COMPUTE_MGMT_PROFILE = ProfileDefinition(
+ {
+ AZURE_COMPUTE_MGMT_PROFILE_TAG: {
+ None: AZURE_COMPUTE_MGMT_CLIENT_API_VERSION,
+ "availability_sets": "2020-12-01",
+ "dedicated_host_groups": "2020-12-01",
+ "dedicated_hosts": "2020-12-01",
+ "disk_accesses": "2020-12-01",
+ "disk_encryption_sets": "2020-12-01",
+ "disk_restore_point": "2020-12-01",
+ "disks": "2020-12-01",
+ "galleries": "2020-09-30",
+ "gallery_application_versions": "2020-09-30",
+ "gallery_applications": "2020-09-30",
+ "gallery_image_versions": "2020-09-30",
+ "gallery_images": "2020-09-30",
+ "gallery_sharing_profile": "2020-09-30",
+ "images": "2020-12-01",
+ "log_analytics": "2020-12-01",
+ "operations": "2020-12-01",
+ "proximity_placement_groups": "2020-12-01",
+ "resource_skus": "2019-04-01",
+ "shared_galleries": "2020-09-30",
+ "shared_gallery_image_versions": "2020-09-30",
+ "shared_gallery_images": "2020-09-30",
+ "snapshots": "2020-12-01",
+ "ssh_public_keys": "2020-12-01",
+ "usage": "2020-12-01",
+ "virtual_machine_extension_images": "2020-12-01",
+ "virtual_machine_extensions": "2020-12-01",
+ "virtual_machine_images": "2020-12-01",
+ "virtual_machine_images_edge_zone": "2020-12-01",
+ "virtual_machine_run_commands": "2020-12-01",
+ "virtual_machine_scale_set_extensions": "2020-12-01",
+ "virtual_machine_scale_set_rolling_upgrades": "2020-12-01",
+ "virtual_machine_scale_set_vm_extensions": "2020-12-01",
+ "virtual_machine_scale_set_vm_run_commands": "2020-12-01",
+ "virtual_machine_scale_set_vms": "2020-12-01",
+ "virtual_machine_scale_sets": "2020-12-01",
+ "virtual_machine_sizes": "2020-12-01",
+ "virtual_machines": "2020-12-01",
+ }
+ },
+ AZURE_COMPUTE_MGMT_PROFILE_TAG + " osm",
+ )
+
+ def __init__(self, vim_account: Dict):
+ self.vim_account = vim_account
+ self.reload_client = True
+ logger = logging.getLogger("azure")
+ logger.setLevel(logging.ERROR)
+ # Store config to create azure subscription later
+ self._config = {
+ "user": vim_account["vim_user"],
+ "passwd": vim_account["vim_password"],
+ "tenant": vim_account["vim_tenant_name"],
+ }
+
+ # SUBSCRIPTION
+ config = vim_account["config"]
+ if "subscription_id" in config:
+ self._config["subscription_id"] = config.get("subscription_id")
+ log.info("Subscription: %s", self._config["subscription_id"])
+ else:
+ log.error("Subscription not specified")
+ return
+
+ # RESOURCE_GROUP
+ if "resource_group" in config:
+ self.resource_group = config.get("resource_group")
+ else:
+ log.error("Azure resource_group is not specified at config")
+ return
+
+ def _reload_connection(self):
+ if self.reload_client:
+ log.debug("reloading azure client")
+ try:
+ self.credentials = ClientSecretCredential(
+ client_id=self._config["user"],
+ client_secret=self._config["passwd"],
+ tenant_id=self._config["tenant"],
+ )
+ self.conn_compute = ComputeManagementClient(
+ self.credentials,
+ self._config["subscription_id"],
+ profile=self.AZURE_COMPUTE_MGMT_PROFILE,
+ )
+ # Set to client created
+ self.reload_client = False
+ except Exception as e:
+ log.error(e)
+
+ def collect_servers_status(self) -> List[Dict]:
+ servers = []
+ log.debug("collect_servers_status")
+ self._reload_connection()
+ try:
+ for vm in self.conn_compute.virtual_machines.list(self.resource_group):
+ id = vm.id
+ array = id.split("/")
+ name = array[-1]
+ status = self.provision_state2osm.get(vm.provisioning_state, "OTHER")
+ if vm.provisioning_state == "Succeeded":
+ # check if machine is running or stopped
+ instance_view = self.conn_compute.virtual_machines.instance_view(
+ self.resource_group, name
+ )
+ for status in instance_view.statuses:
+ splitted_status = status.code.split("/")
+ if (
+ len(splitted_status) == 2
+ and splitted_status[0] == "PowerState"
+ ):
+ status = self.power_state2osm.get(
+ splitted_status[1], "OTHER"
+ )
+ # log.info(f'id: {id}, name: {name}, status: {status}')
+ vm = {
+ "id": id,
+ "name": name,
+ "status": (1 if (status == "ACTIVE") else 0),
+ }
+ servers.append(vm)
+ except Exception as e:
+ log.error(e)
+ return servers
+
+ def is_vim_ok(self) -> bool:
+ status = False
+ self.reload_client = True
+ try:
+ self._reload_connection()
+ status = True
+ except Exception as e:
+ log.error(e)
+ return status
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+from typing import Dict, List
+
+
+class VIMConnector:
+ def __init__(self, vim_account: Dict):
+ pass
+
+ # def collect_servers_status(self) -> List[Metric]:
+ def collect_servers_status(self) -> List:
+ pass
+
+ def is_vim_ok(self) -> bool:
+ pass
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+# pylint: disable=E1101
+
+import logging
+from typing import Dict, List
+
+from google.oauth2 import service_account
+import googleapiclient.discovery
+from osm_mon.vim_connectors.base_vim import VIMConnector
+
+log = logging.getLogger(__name__)
+
+
+class GcpCollector(VIMConnector):
+ def __init__(self, vim_account: Dict):
+ self.vim_account = vim_account
+ self.project = vim_account["vim_tenant_name"] or vim_account["vim_tenant_id"]
+
+ # REGION - Google Cloud considers regions and zones. A specific region
+ # can have more than one zone (for instance: region us-west1 with the
+ # zones us-west1-a, us-west1-b and us-west1-c). So the region name
+ # specified in the config will be considered as a specific zone for GC
+ # and the region will be calculated from that without the preffix.
+ if "config" in vim_account:
+ config = vim_account["config"]
+ if "region_name" in config:
+ self.zone = config.get("region_name")
+ self.region = self.zone.rsplit("-", 1)[0]
+ else:
+ log.error("Google Cloud region_name not specified in config")
+ else:
+ log.error("config is not specified in VIM")
+
+ # Credentials
+ scopes = ["https://www.googleapis.com/auth/cloud-platform"]
+ self.credentials = None
+ if "credentials" in config:
+ log.debug("Setting credentials")
+ # Settings Google Cloud credentials dict
+ creds_body = config["credentials"]
+ creds = service_account.Credentials.from_service_account_info(creds_body)
+ if "sa_file" in config:
+ creds = service_account.Credentials.from_service_account_file(
+ config.get("sa_file"), scopes=scopes
+ )
+ log.debug("Credentials: %s", creds)
+ # Construct a Resource for interacting with an API.
+ self.credentials = creds
+ try:
+ self.conn_compute = googleapiclient.discovery.build(
+ "compute", "v1", credentials=creds
+ )
+ except Exception as e:
+ log.error(e)
+ else:
+ log.error("It is not possible to init GCP with no credentials")
+
+ def collect_servers_status(self) -> List[Dict]:
+ servers = []
+ try:
+ response = (
+ self.conn_compute.instances()
+ .list(project=self.project, zone=self.zone)
+ .execute()
+ )
+ if "items" in response:
+ log.info(response["items"])
+ for server in response["items"]:
+ vm = {
+ "id": server["id"],
+ "name": server["name"],
+ "status": (1 if (server["status"] == "RUNNING") else 0),
+ }
+ servers.append(vm)
+ except Exception as e:
+ log.error(e)
+ return servers
+
+ def is_vim_ok(self) -> bool:
+ status = False
+ try:
+ self.conn_compute.zones().get(
+ project=self.project, zone=self.zone
+ ).execute()
+ status = True
+ except Exception as e:
+ log.error(e)
+ return status
--- /dev/null
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+import logging
+from typing import Dict, List
+
+from keystoneauth1 import session
+from keystoneauth1.identity import v3
+from novaclient import client as nova_client
+from osm_mon.vim_connectors.base_vim import VIMConnector
+
+log = logging.getLogger(__name__)
+
+
+class CertificateNotCreated(Exception):
+ pass
+
+
+class OpenStackCollector(VIMConnector):
+ def __init__(self, vim_account: Dict):
+ log.info("__init__")
+ self.vim_account = vim_account
+ self.vim_session = None
+ self.vim_session = self._get_session(vim_account)
+ self.nova = self._build_nova_client()
+
+ def _get_session(self, creds: Dict):
+ verify_ssl = True
+ project_domain_name = "Default"
+ user_domain_name = "Default"
+ try:
+ if "config" in creds:
+ vim_config = creds["config"]
+ if "insecure" in vim_config and vim_config["insecure"]:
+ verify_ssl = False
+ if "ca_cert" in vim_config:
+ verify_ssl = vim_config["ca_cert"]
+ elif "ca_cert_content" in vim_config:
+ # vim_config = self._create_file_cert(vim_config, creds["_id"])
+ verify_ssl = vim_config["ca_cert"]
+ if "project_domain_name" in vim_config:
+ project_domain_name = vim_config["project_domain_name"]
+ if "user_domain_name" in vim_config:
+ user_domain_name = vim_config["user_domain_name"]
+ auth = v3.Password(
+ auth_url=creds["vim_url"],
+ username=creds["vim_user"],
+ password=creds["vim_password"],
+ project_name=creds["vim_tenant_name"],
+ project_domain_name=project_domain_name,
+ user_domain_name=user_domain_name,
+ )
+ return session.Session(auth=auth, verify=verify_ssl, timeout=10)
+ except CertificateNotCreated as e:
+ log.error(e)
+
+ def _build_nova_client(self) -> nova_client.Client:
+ return nova_client.Client("2", session=self.vim_session, timeout=10)
+
+ def collect_servers_status(self) -> List[Dict]:
+ log.info("collect_servers_status")
+ servers = []
+ for server in self.nova.servers.list(detailed=True):
+ vm = {
+ "id": server.id,
+ "name": server.name,
+ "status": (0 if (server.status == "ERROR") else 1),
+ }
+ servers.append(vm)
+ return servers
+
+ def is_vim_ok(self) -> bool:
+ try:
+ self.nova.servers.list()
+ return True
+ except Exception as e:
+ log.warning("VIM status is not OK: %s" % e)
+ return False
done"
+#######################################################################################
+[testenv:dist]
+deps = {[testenv]deps}
+ -r{toxinidir}/requirements-dist.txt
+
+# In the commands, we copy the requirements.txt to be presented as a source file (.py)
+# so it gets included in the .deb package for others to consume
+commands =
+ sh -c 'cp requirements.txt src/osm_ngsa/requirements.txt'
+ sh -c 'cp README.rst src/osm_ngsa/README.rst'
+ python3 setup.py --command-packages=stdeb.command sdist_dsc
+ sh -c 'cd deb_dist/osm-ngsa*/ && dpkg-buildpackage -rfakeroot -uc -us'
+ sh -c 'rm src/osm_ngsa/requirements.txt'
+ sh -c 'rm src/osm_ngsa/README.rst'
+whitelist_externals = sh
+
+
#######################################################################################
[flake8]
ignore =
projects / osm / NG-SA.git / commitdiff