X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_lcm%2Flcm.py;h=6c8be98c72d04b95816e0e390c03bfeb643d9310;hb=a5ae90b046fc9760c542832e4543d4c4790f869a;hp=365272bb4349aabac126864fd263511a6ebacc2a;hpb=744303e157182880777f169259ea7cd4431af1d7;p=osm%2FLCM.git diff --git a/osm_lcm/lcm.py b/osm_lcm/lcm.py index 365272b..6c8be98 100644 --- a/osm_lcm/lcm.py +++ b/osm_lcm/lcm.py @@ -29,23 +29,25 @@ import logging.handlers import getopt import sys -from osm_lcm import ns -from osm_lcm import vim_sdn -from osm_lcm import netslice -from osm_lcm import ROclient +from osm_lcm import ns, prometheus, vim_sdn, netslice +from osm_lcm.ng_ro import NgRoException, NgRoClient +from osm_lcm.ROclient import ROClient, ROClientException -from time import time, sleep +from time import time from osm_lcm.lcm_utils import versiontuple, LcmException, TaskRegistry, LcmExceptionExit -from osm_lcm import version as _lcm_version, version_date as lcm_version_date +from osm_lcm import version as lcm_version, version_date as lcm_version_date -from osm_common import dbmemory, dbmongo, fslocal, fsmongo, msglocal, msgkafka +from osm_common import msglocal, msgkafka from osm_common import version as common_version from osm_common.dbbase import DbException from osm_common.fsbase import FsException from osm_common.msgbase import MsgException +from osm_lcm.data_utils.database.database import Database +from osm_lcm.data_utils.filesystem.filesystem import Filesystem from os import environ, path from random import choice as random_choice from n2vc import version as n2vc_version +import traceback if os.getenv('OSMLCM_PDB_DEBUG', None) is not None: pdb.set_trace() @@ -56,7 +58,6 @@ min_RO_version = "6.0.2" min_n2vc_version = "0.0.2" min_common_version = "0.1.19" -lcm_version = _lcm_version health_check_file = path.expanduser("~") + "/time_last_ping" # TODO find better location for this file @@ -64,6 +65,8 @@ class Lcm: ping_interval_pace = 120 # how many time ping is send once is confirmed all is running ping_interval_boot = 5 # how many time ping is sent when booting + cfg_logger_name = {"message": "lcm.msg", "database": "lcm.db", "storage": "lcm.fs", "tsdb": "lcm.prometheus"} + # ^ contains for each section at lcm.cfg the used logger name def __init__(self, config_file, loop=None): """ @@ -71,7 +74,6 @@ class Lcm: :param config: two level dictionary with configuration. Top level should contain 'database', 'storage', :return: None """ - global lcm_version self.db = None self.msg = None self.msg_admin = None @@ -88,13 +90,21 @@ class Lcm: config = self.read_config_file(config_file) self.config = config self.config["ro_config"] = { - "endpoint_url": "http://{}:{}/openmano".format(config["RO"]["host"], config["RO"]["port"]), + "ng": config["RO"].get("ng", False), + "uri": config["RO"].get("uri"), "tenant": config.get("tenant", "osm"), - "logger_name": "lcm.ROclient", - "loglevel": "ERROR", + "logger_name": "lcm.roclient", + "loglevel": config["RO"].get("loglevel", "ERROR"), } + if not self.config["ro_config"]["uri"]: + self.config["ro_config"]["uri"] = "http://{}:{}/".format(config["RO"]["host"], config["RO"]["port"]) + elif "/ro" in self.config["ro_config"]["uri"][-4:] or "/openmano" in self.config["ro_config"]["uri"][-10:]: + # uri ends with '/ro', '/ro/', '/openmano', '/openmano/' + index = self.config["ro_config"]["uri"][-1].rfind("/") + self.config["ro_config"]["uri"] = self.config["ro_config"]["uri"][index+1] self.loop = loop or asyncio.get_event_loop() + self.ns = self.netslice = self.vim = self.wim = self.sdn = self.k8scluster = self.k8srepo = None # logging log_format_simple = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(message)s" @@ -116,7 +126,7 @@ class Lcm: self.logger.setLevel(config["global"]["loglevel"]) # logging other modules - for k1, logname in {"message": "lcm.msg", "database": "lcm.db", "storage": "lcm.fs"}.items(): + for k1, logname in self.cfg_logger_name.items(): config[k1]["logger_name"] = logname logger_module = logging.getLogger(logname) if config[k1].get("logfile"): @@ -140,26 +150,9 @@ class Lcm: common_version, min_common_version)) try: - # TODO check database version - if config["database"]["driver"] == "mongo": - self.db = dbmongo.DbMongo() - self.db.db_connect(config["database"]) - elif config["database"]["driver"] == "memory": - self.db = dbmemory.DbMemory() - self.db.db_connect(config["database"]) - else: - raise LcmException("Invalid configuration param '{}' at '[database]':'driver'".format( - config["database"]["driver"])) - - if config["storage"]["driver"] == "local": - self.fs = fslocal.FsLocal() - self.fs.fs_connect(config["storage"]) - elif config["storage"]["driver"] == "mongo": - self.fs = fsmongo.FsMongo() - self.fs.fs_connect(config["storage"]) - else: - raise LcmException("Invalid configuration param '{}' at '[storage]':'driver'".format( - config["storage"]["driver"])) + self.db = Database(config).instance.db + + self.fs = Filesystem(config).instance.fs # copy message configuration in order to remove 'group_id' for msg_admin config_message = config["message"].copy() @@ -184,32 +177,45 @@ class Lcm: raise LcmException(str(e)) # contains created tasks/futures to be able to cancel - self.lcm_tasks = TaskRegistry(self.worker_id, self.db, self.logger) + self.lcm_tasks = TaskRegistry(self.worker_id, self.logger) - self.ns = ns.NsLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop) - self.netslice = netslice.NetsliceLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop) - self.vim = vim_sdn.VimLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop) - self.wim = vim_sdn.WimLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop) - self.sdn = vim_sdn.SdnLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop) - self.k8scluster = vim_sdn.K8sClusterLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop) - self.k8srepo = vim_sdn.K8sRepoLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.config, self.loop) + if self.config.get("tsdb") and self.config["tsdb"].get("driver"): + if self.config["tsdb"]["driver"] == "prometheus": + self.prometheus = prometheus.Prometheus(self.config["tsdb"], self.worker_id, self.loop) + else: + raise LcmException("Invalid configuration param '{}' at '[tsdb]':'driver'".format( + config["tsdb"]["driver"])) + else: + self.prometheus = None async def check_RO_version(self): tries = 14 last_error = None while True: + ro_uri = self.config["ro_config"]["uri"] try: - ro_server = ROclient.ROClient(self.loop, **self.config["ro_config"]) - ro_version = await ro_server.get_version() + # try new RO, if fail old RO + try: + self.config["ro_config"]["uri"] = ro_uri + "ro" + ro_server = NgRoClient(self.loop, **self.config["ro_config"]) + ro_version = await ro_server.get_version() + self.config["ro_config"]["ng"] = True + except Exception: + self.config["ro_config"]["uri"] = ro_uri + "openmano" + ro_server = ROClient(self.loop, **self.config["ro_config"]) + ro_version = await ro_server.get_version() + self.config["ro_config"]["ng"] = False if versiontuple(ro_version) < versiontuple(min_RO_version): raise LcmException("Not compatible osm/RO version '{}'. Needed '{}' or higher".format( ro_version, min_RO_version)) - self.logger.info("Connected to RO version {}".format(ro_version)) + self.logger.info("Connected to RO version {} new-generation version {}". + format(ro_version, self.config["ro_config"]["ng"])) return - except ROclient.ROClientException as e: + except (ROClientException, NgRoException) as e: + self.config["ro_config"]["uri"] = ro_uri tries -= 1 - error_text = "Error while connecting to RO on {}: {}".format(self.config["ro_config"]["endpoint_url"], - e) + traceback.print_tb(e.__traceback__) + error_text = "Error while connecting to RO on {}: {}".format(self.config["ro_config"]["uri"], e) if tries <= 0: self.logger.critical(error_text) raise LcmException(error_text) @@ -222,7 +228,6 @@ class Lcm: self.logger.debug("Starting/Ending test task: {}".format(param)) async def kafka_ping(self): - global lcm_version self.logger.debug("Task kafka_ping Enter") consecutive_errors = 0 first_start = True @@ -289,6 +294,10 @@ class Lcm: except Exception as e: self.logger.error("Cannot write into '{}' for healthcheck: {}".format(health_check_file, e)) return + elif topic == "pla": + if command == "placement": + self.ns.update_nsrs_with_pla_result(params) + return elif topic == "k8scluster": if command == "create" or command == "created": k8scluster_id = params.get("_id") @@ -313,7 +322,7 @@ class Lcm: self.lcm_tasks.register("k8srepo", k8srepo_id, order_id, "k8srepo_delete", task) return elif topic == "ns": - if command == "instantiate" or command == "instantiated": + if command == "instantiate": # self.logger.debug("Deploying NS {}".format(nsr_id)) nslcmop = params nslcmop_id = nslcmop["_id"] @@ -321,7 +330,7 @@ class Lcm: task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id)) self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task) return - elif command == "terminate" or command == "terminated": + elif command == "terminate": # self.logger.debug("Deleting NS {}".format(nsr_id)) nslcmop = params nslcmop_id = nslcmop["_id"] @@ -364,7 +373,7 @@ class Lcm: elif command in ("terminated", "instantiated", "scaled", "actioned"): # "scaled-cooldown-time" return elif topic == "nsi": # netslice LCM processes (instantiate, terminate, etc) - if command == "instantiate" or command == "instantiated": + if command == "instantiate": # self.logger.debug("Instantiating Network Slice {}".format(nsilcmop["netsliceInstanceId"])) nsilcmop = params nsilcmop_id = nsilcmop["_id"] # slice operation id @@ -372,7 +381,7 @@ class Lcm: task = asyncio.ensure_future(self.netslice.instantiate(nsir_id, nsilcmop_id)) self.lcm_tasks.register("nsi", nsir_id, nsilcmop_id, "nsi_instantiate", task) return - elif command == "terminate" or command == "terminated": + elif command == "terminate": # self.logger.debug("Terminating Network Slice NS {}".format(nsilcmop["netsliceInstanceId"])) nsilcmop = params nsilcmop_id = nsilcmop["_id"] # slice operation id @@ -401,8 +410,9 @@ class Lcm: elif topic == "vim_account": vim_id = params["_id"] if command in ("create", "created"): - task = asyncio.ensure_future(self.vim.create(params, order_id)) - self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_create", task) + if not self.config["ro_config"].get("ng"): + task = asyncio.ensure_future(self.vim.create(params, order_id)) + self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_create", task) return elif command == "delete" or command == "deleted": self.lcm_tasks.cancel(topic, vim_id) @@ -414,16 +424,18 @@ class Lcm: sys.stdout.flush() return elif command in ("edit", "edited"): - task = asyncio.ensure_future(self.vim.edit(params, order_id)) - self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_edit", task) + if not self.config["ro_config"].get("ng"): + task = asyncio.ensure_future(self.vim.edit(params, order_id)) + self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_edit", task) return elif command == "deleted": return # TODO cleaning of task just in case should be done elif topic == "wim_account": wim_id = params["_id"] if command in ("create", "created"): - task = asyncio.ensure_future(self.wim.create(params, order_id)) - self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_create", task) + if not self.config["ro_config"].get("ng"): + task = asyncio.ensure_future(self.wim.create(params, order_id)) + self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_create", task) return elif command == "delete" or command == "deleted": self.lcm_tasks.cancel(topic, wim_id) @@ -443,8 +455,9 @@ class Lcm: elif topic == "sdn": _sdn_id = params["_id"] if command in ("create", "created"): - task = asyncio.ensure_future(self.sdn.create(params, order_id)) - self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_create", task) + if not self.config["ro_config"].get("ng"): + task = asyncio.ensure_future(self.sdn.create(params, order_id)) + self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_create", task) return elif command == "delete" or command == "deleted": self.lcm_tasks.cancel(topic, _sdn_id) @@ -466,10 +479,10 @@ class Lcm: self.first_start = True while self.consecutive_errors < 10: try: - topics = ("ns", "vim_account", "wim_account", "sdn", "nsi", "k8scluster", "k8srepo") + topics = ("ns", "vim_account", "wim_account", "sdn", "nsi", "k8scluster", "k8srepo", "pla") topics_admin = ("admin", ) await asyncio.gather( - self.msg.aioread(topics, self.loop, self.kafka_read_callback), + self.msg.aioread(topics, self.loop, self.kafka_read_callback, from_beginning=True), self.msg_admin.aioread(topics_admin, self.loop, self.kafka_read_callback, group_id=False) ) @@ -495,6 +508,19 @@ class Lcm: # check RO version self.loop.run_until_complete(self.check_RO_version()) + self.ns = ns.NsLcm(self.msg, self.lcm_tasks, self.config, self.loop, self.prometheus) + self.netslice = netslice.NetsliceLcm(self.msg, self.lcm_tasks, self.config, self.loop, + self.ns) + self.vim = vim_sdn.VimLcm(self.msg, self.lcm_tasks, self.config, self.loop) + self.wim = vim_sdn.WimLcm(self.msg, self.lcm_tasks, self.config, self.loop) + self.sdn = vim_sdn.SdnLcm(self.msg, self.lcm_tasks, self.config, self.loop) + self.k8scluster = vim_sdn.K8sClusterLcm(self.msg, self.lcm_tasks, self.config, self.loop) + self.k8srepo = vim_sdn.K8sRepoLcm(self.msg, self.lcm_tasks, self.config, self.loop) + + # configure tsdb prometheus + if self.prometheus: + self.loop.run_until_complete(self.prometheus.start()) + self.loop.run_until_complete(asyncio.gather( self.kafka_read(), self.kafka_ping() @@ -597,18 +623,6 @@ class Lcm: return ''.join(random_choice("0123456789abcdef") for _ in range(12)) -def _get_version(): - """ - Try to get version from package using pkg_resources (available with setuptools) - """ - global lcm_version - try: - from pkg_resources import get_distribution - lcm_version = get_distribution("osm_lcm").version - except Exception: - pass - - def usage(): print("""Usage: {} [options] -c|--config [configuration_file]: loads the configuration file (default: ./lcm.cfg) @@ -619,27 +633,10 @@ def usage(): # --log-socket-port PORT: send logs using this port (default: 9022)") -def health_check(): - retry = 2 - while retry: - retry -= 1 - try: - with open(health_check_file, "r") as f: - last_received_ping = f.read() - - if time() - float(last_received_ping) < Lcm.ping_interval_pace + 10: - exit(0) - except Exception: - pass - if retry: - sleep(6) - exit(1) - - if __name__ == '__main__': try: - print("SYS.PATH='{}'".format(sys.path)) + # print("SYS.PATH='{}'".format(sys.path)) # load parameters and configuration # -h # -c value @@ -656,7 +653,8 @@ if __name__ == '__main__': elif o in ("-c", "--config"): config_file = a elif o == "--health-check": - health_check() + from osm_lcm.lcm_hc import health_check + health_check(health_check_file, Lcm.ping_interval_pace) # elif o == "--log-socket-port": # log_socket_port = a # elif o == "--log-socket-host": @@ -677,8 +675,6 @@ if __name__ == '__main__': else: print("No configuration file 'lcm.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr) exit(1) - # get version from package and upate global lcm_version - _get_version() lcm = Lcm(config_file) lcm.start() except (LcmException, getopt.GetoptError) as e: