X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_lcm%2Flcm.py;h=de829bd5c3d2733b227ff78f3e7d1db3bf3c8373;hb=5899e08821d446ce8e036aa59a3451487b1bd0b3;hp=c4e5eed8bb3948af92b4431502be67f7e8240e30;hpb=9597c29234cffe76f9ae2bd7bcaa0955e010b4b4;p=osm%2FLCM.git diff --git a/osm_lcm/lcm.py b/osm_lcm/lcm.py index c4e5eed..de829bd 100644 --- a/osm_lcm/lcm.py +++ b/osm_lcm/lcm.py @@ -17,21 +17,28 @@ # under the License. ## + +# DEBUG WITH PDB +import os +import pdb + import asyncio import yaml import logging import logging.handlers import getopt import sys -import ROclient -import ns -import vim_sdn -import netslice + +from osm_lcm import ns +from osm_lcm import vim_sdn +from osm_lcm import netslice +from osm_lcm import ROclient + from time import time, sleep -from lcm_utils import versiontuple, LcmException, TaskRegistry, LcmExceptionExit +from osm_lcm.lcm_utils import versiontuple, LcmException, TaskRegistry, LcmExceptionExit +from osm_lcm import version as lcm_version, version_date as lcm_version_date -# from osm_lcm import version as lcm_version, version_date as lcm_version_date, ROclient -from osm_common import dbmemory, dbmongo, fslocal, msglocal, msgkafka +from osm_common import dbmemory, dbmongo, fslocal, fsmongo, msglocal, msgkafka from osm_common import version as common_version from osm_common.dbbase import DbException from osm_common.fsbase import FsException @@ -40,14 +47,18 @@ from os import environ, path from random import choice as random_choice from n2vc import version as n2vc_version +if os.getenv('OSMLCM_PDB_DEBUG', None) is not None: + pdb.set_trace() + __author__ = "Alfonso Tierno" -min_RO_version = [0, 6, 3] +min_RO_version = "6.0.2" min_n2vc_version = "0.0.2" + min_common_version = "0.1.19" # uncomment if LCM is installed as library and installed, and get them from __init__.py -lcm_version = '0.1.37' -lcm_version_date = '2019-04-30' +# lcm_version = '0.1.41' +# lcm_version_date = '2019-06-19' health_check_file = path.expanduser("~") + "/time_last_ping" # TODO find better location for this file @@ -71,8 +82,6 @@ class Lcm: self.consecutive_errors = 0 self.first_start = False - # contains created tasks/futures to be able to cancel - self.lcm_tasks = TaskRegistry() # logging self.logger = logging.getLogger('lcm') # get id @@ -149,10 +158,14 @@ class Lcm: if config["storage"]["driver"] == "local": self.fs = fslocal.FsLocal() self.fs.fs_connect(config["storage"]) + elif config["storage"]["driver"] == "mongo": + self.fs = fsmongo.FsMongo() + self.fs.fs_connect(config["storage"]) else: raise LcmException("Invalid configuration param '{}' at '[storage]':'driver'".format( config["storage"]["driver"])) + # copy message configuration in order to remove 'group_id' for msg_admin config_message = config["message"].copy() config_message["loop"] = self.loop if config_message["driver"] == "local": @@ -174,25 +187,40 @@ class Lcm: self.logger.critical(str(e), exc_info=True) raise LcmException(str(e)) + # contains created tasks/futures to be able to cancel + self.lcm_tasks = TaskRegistry(self.worker_id, self.db, self.logger) + self.ns = ns.NsLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.ro_config, self.vca_config, self.loop) self.netslice = netslice.NetsliceLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.ro_config, self.vca_config, self.loop) self.vim = vim_sdn.VimLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.ro_config, self.loop) self.wim = vim_sdn.WimLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.ro_config, self.loop) self.sdn = vim_sdn.SdnLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.ro_config, self.loop) + self.k8scluster = vim_sdn.K8sClusterLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.vca_config, self.loop) + self.k8srepo = vim_sdn.K8sRepoLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.vca_config, self.loop) async def check_RO_version(self): - try: - RO = ROclient.ROClient(self.loop, **self.ro_config) - RO_version = await RO.get_version() - if RO_version < min_RO_version: - raise LcmException("Not compatible osm/RO version '{}.{}.{}'. Needed '{}.{}.{}' or higher".format( - *RO_version, *min_RO_version - )) - except ROclient.ROClientException as e: - error_text = "Error while conneting to osm/RO " + str(e) - self.logger.critical(error_text, exc_info=True) - raise LcmException(error_text) + tries = 14 + last_error = None + while True: + try: + ro_server = ROclient.ROClient(self.loop, **self.ro_config) + ro_version = await ro_server.get_version() + if versiontuple(ro_version) < versiontuple(min_RO_version): + raise LcmException("Not compatible osm/RO version '{}'. Needed '{}' or higher".format( + ro_version, min_RO_version)) + self.logger.info("Connected to RO version {}".format(ro_version)) + return + except ROclient.ROClientException as e: + tries -= 1 + error_text = "Error while connecting to RO on {}: {}".format(self.ro_config["endpoint_url"], e) + if tries <= 0: + self.logger.critical(error_text) + raise LcmException(error_text) + if last_error != error_text: + last_error = error_text + self.logger.error(error_text + ". Waiting until {} seconds".format(5*tries)) + await asyncio.sleep(5) async def test(self, param=None): self.logger.debug("Starting/Ending test task: {}".format(param)) @@ -264,8 +292,31 @@ class Lcm: except Exception as e: self.logger.error("Cannot write into '{}' for healthcheck: {}".format(health_check_file, e)) return + elif topic == "k8scluster": + if command == "create" or command == "created": + k8scluster_id = params.get("_id") + task = asyncio.ensure_future(self.k8scluster.create(params, order_id)) + self.lcm_tasks.register("k8scluster", k8scluster_id, order_id, "k8scluster_create", task) + return + elif command == "delete" or command == "deleted": + k8scluster_id = params.get("_id") + task = asyncio.ensure_future(self.k8scluster.delete(params, order_id)) + self.lcm_tasks.register("k8scluster", k8scluster_id, order_id, "k8scluster_delete", task) + return + elif topic == "k8srepo": + if command == "create" or command == "created": + k8srepo_id = params.get("_id") + self.logger.debug("k8srepo_id = {}".format(k8srepo_id)) + task = asyncio.ensure_future(self.k8srepo.create(params, order_id)) + self.lcm_tasks.register("k8srepo", k8srepo_id, order_id, "k8srepo_create", task) + return + elif command == "delete" or command == "deleted": + k8srepo_id = params.get("_id") + task = asyncio.ensure_future(self.k8srepo.delete(params, order_id)) + self.lcm_tasks.register("k8srepo", k8srepo_id, order_id, "k8srepo_delete", task) + return elif topic == "ns": - if command == "instantiate": + if command == "instantiate" or command == "instantiated": # self.logger.debug("Deploying NS {}".format(nsr_id)) nslcmop = params nslcmop_id = nslcmop["_id"] @@ -273,7 +324,7 @@ class Lcm: task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id)) self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task) return - elif command == "terminate": + elif command == "terminate" or command == "terminated": # self.logger.debug("Deleting NS {}".format(nsr_id)) nslcmop = params nslcmop_id = nslcmop["_id"] @@ -316,7 +367,7 @@ class Lcm: elif command in ("terminated", "instantiated", "scaled", "actioned"): # "scaled-cooldown-time" return elif topic == "nsi": # netslice LCM processes (instantiate, terminate, etc) - if command == "instantiate": + if command == "instantiate" or command == "instantiated": # self.logger.debug("Instantiating Network Slice {}".format(nsilcmop["netsliceInstanceId"])) nsilcmop = params nsilcmop_id = nsilcmop["_id"] # slice operation id @@ -324,7 +375,7 @@ class Lcm: task = asyncio.ensure_future(self.netslice.instantiate(nsir_id, nsilcmop_id)) self.lcm_tasks.register("nsi", nsir_id, nsilcmop_id, "nsi_instantiate", task) return - elif command == "terminate": + elif command == "terminate" or command == "terminated": # self.logger.debug("Terminating Network Slice NS {}".format(nsilcmop["netsliceInstanceId"])) nsilcmop = params nsilcmop_id = nsilcmop["_id"] # slice operation id @@ -352,57 +403,63 @@ class Lcm: return elif topic == "vim_account": vim_id = params["_id"] - if command == "create": + if command in ("create", "created"): task = asyncio.ensure_future(self.vim.create(params, order_id)) self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_create", task) return - elif command == "delete": + elif command == "delete" or command == "deleted": self.lcm_tasks.cancel(topic, vim_id) - task = asyncio.ensure_future(self.vim.delete(vim_id, order_id)) + task = asyncio.ensure_future(self.vim.delete(params, order_id)) self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_delete", task) return elif command == "show": print("not implemented show with vim_account") sys.stdout.flush() return - elif command == "edit": + elif command in ("edit", "edited"): task = asyncio.ensure_future(self.vim.edit(params, order_id)) self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_edit", task) return + elif command == "deleted": + return # TODO cleaning of task just in case should be done elif topic == "wim_account": wim_id = params["_id"] - if command == "create": + if command in ("create", "created"): task = asyncio.ensure_future(self.wim.create(params, order_id)) self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_create", task) return - elif command == "delete": + elif command == "delete" or command == "deleted": self.lcm_tasks.cancel(topic, wim_id) - task = asyncio.ensure_future(self.wim.delete(wim_id, order_id)) + task = asyncio.ensure_future(self.wim.delete(params, order_id)) self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_delete", task) return elif command == "show": print("not implemented show with wim_account") sys.stdout.flush() return - elif command == "edit": + elif command in ("edit", "edited"): task = asyncio.ensure_future(self.wim.edit(params, order_id)) self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_edit", task) return + elif command == "deleted": + return # TODO cleaning of task just in case should be done elif topic == "sdn": _sdn_id = params["_id"] - if command == "create": + if command in ("create", "created"): task = asyncio.ensure_future(self.sdn.create(params, order_id)) self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_create", task) return - elif command == "delete": + elif command == "delete" or command == "deleted": self.lcm_tasks.cancel(topic, _sdn_id) - task = asyncio.ensure_future(self.sdn.delete(_sdn_id, order_id)) + task = asyncio.ensure_future(self.sdn.delete(params, order_id)) self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_delete", task) return - elif command == "edit": + elif command in ("edit", "edited"): task = asyncio.ensure_future(self.sdn.edit(params, order_id)) self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_edit", task) return + elif command == "deleted": + return # TODO cleaning of task just in case should be done self.logger.critical("unknown topic {} and command '{}'".format(topic, command)) async def kafka_read(self): @@ -412,7 +469,7 @@ class Lcm: self.first_start = True while self.consecutive_errors < 10: try: - topics = ("ns", "vim_account", "wim_account", "sdn", "nsi") + topics = ("ns", "vim_account", "wim_account", "sdn", "nsi", "k8scluster", "k8srepo") topics_admin = ("admin", ) await asyncio.gather( self.msg.aioread(topics, self.loop, self.kafka_read_callback), @@ -472,7 +529,7 @@ class Lcm: # and not parse integer or boolean try: with open(config_file) as f: - conf = yaml.load(f) + conf = yaml.load(f, Loader=yaml.Loader) for k, v in environ.items(): if not k.startswith("OSMLCM_"): continue @@ -521,7 +578,7 @@ class Lcm: def usage(): print("""Usage: {} [options] - -c|--config [configuration_file]: loads the configuration file (default: ./nbi.cfg) + -c|--config [configuration_file]: loads the configuration file (default: ./lcm.cfg) --health-check: do not run lcm, but inspect kafka bus to determine if lcm is healthy -h|--help: shows this help """.format(sys.argv[0])) @@ -547,8 +604,15 @@ def health_check(): if __name__ == '__main__': + try: + print("SYS.PATH='{}'".format(sys.path)) # load parameters and configuration + # -h + # -c value + # --config value + # --help + # --health-check opts, args = getopt.getopt(sys.argv[1:], "hc:", ["config=", "help", "health-check"]) # TODO add "log-socket-host=", "log-socket-port=", "log-file=" config_file = None @@ -568,9 +632,10 @@ if __name__ == '__main__': # log_file = a else: assert False, "Unhandled option" + if config_file: if not path.isfile(config_file): - print("configuration file '{}' not exist".format(config_file), file=sys.stderr) + print("configuration file '{}' does not exist".format(config_file), file=sys.stderr) exit(1) else: for config_file in (__file__[:__file__.rfind(".")] + ".cfg", "./lcm.cfg", "/etc/osm/lcm.cfg"):