X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_lcm%2Flcm.py;h=c4e5eed8bb3948af92b4431502be67f7e8240e30;hb=9597c29234cffe76f9ae2bd7bcaa0955e010b4b4;hp=ff90ff112808258b4928548b65c11d4e7e00bb42;hpb=8a0d6caa091700066ed8ceed1cda8495df547909;p=osm%2FLCM.git diff --git a/osm_lcm/lcm.py b/osm_lcm/lcm.py index ff90ff1..c4e5eed 100644 --- a/osm_lcm/lcm.py +++ b/osm_lcm/lcm.py @@ -1,6 +1,22 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- +## +# Copyright 2018 Telefonica S.A. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +## + import asyncio import yaml import logging @@ -11,7 +27,8 @@ import ROclient import ns import vim_sdn import netslice -from lcm_utils import versiontuple, LcmException, TaskRegistry +from time import time, sleep +from lcm_utils import versiontuple, LcmException, TaskRegistry, LcmExceptionExit # from osm_lcm import version as lcm_version, version_date as lcm_version_date, ROclient from osm_common import dbmemory, dbmongo, fslocal, msglocal, msgkafka @@ -20,16 +37,18 @@ from osm_common.dbbase import DbException from osm_common.fsbase import FsException from osm_common.msgbase import MsgException from os import environ, path +from random import choice as random_choice from n2vc import version as n2vc_version __author__ = "Alfonso Tierno" -min_RO_version = [0, 5, 72] +min_RO_version = [0, 6, 3] min_n2vc_version = "0.0.2" -min_common_version = "0.1.11" +min_common_version = "0.1.19" # uncomment if LCM is installed as library and installed, and get them from __init__.py -lcm_version = '0.1.24' -lcm_version_date = '2018-11-16' +lcm_version = '0.1.37' +lcm_version_date = '2019-04-30' +health_check_file = path.expanduser("~") + "/time_last_ping" # TODO find better location for this file class Lcm: @@ -46,13 +65,18 @@ class Lcm: self.db = None self.msg = None + self.msg_admin = None self.fs = None self.pings_not_received = 1 + self.consecutive_errors = 0 + self.first_start = False # contains created tasks/futures to be able to cancel self.lcm_tasks = TaskRegistry() # logging self.logger = logging.getLogger('lcm') + # get id + self.worker_id = self.get_process_id() # load configuration config = self.read_config_file(config_file) self.config = config @@ -129,15 +153,23 @@ class Lcm: raise LcmException("Invalid configuration param '{}' at '[storage]':'driver'".format( config["storage"]["driver"])) - if config["message"]["driver"] == "local": + config_message = config["message"].copy() + config_message["loop"] = self.loop + if config_message["driver"] == "local": self.msg = msglocal.MsgLocal() - self.msg.connect(config["message"]) - elif config["message"]["driver"] == "kafka": + self.msg.connect(config_message) + self.msg_admin = msglocal.MsgLocal() + config_message.pop("group_id", None) + self.msg_admin.connect(config_message) + elif config_message["driver"] == "kafka": self.msg = msgkafka.MsgKafka() - self.msg.connect(config["message"]) + self.msg.connect(config_message) + self.msg_admin = msgkafka.MsgKafka() + config_message.pop("group_id", None) + self.msg_admin.connect(config_message) else: raise LcmException("Invalid configuration param '{}' at '[message]':'driver'".format( - config["storage"]["driver"])) + config["message"]["driver"])) except (DbException, FsException, MsgException) as e: self.logger.critical(str(e), exc_info=True) raise LcmException(str(e)) @@ -146,6 +178,7 @@ class Lcm: self.netslice = netslice.NetsliceLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.ro_config, self.vca_config, self.loop) self.vim = vim_sdn.VimLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.ro_config, self.loop) + self.wim = vim_sdn.WimLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.ro_config, self.loop) self.sdn = vim_sdn.SdnLcm(self.db, self.msg, self.fs, self.lcm_tasks, self.ro_config, self.loop) async def check_RO_version(self): @@ -172,7 +205,10 @@ class Lcm: self.pings_not_received = 1 while True: try: - await self.msg.aiowrite("admin", "ping", {"from": "lcm", "to": "lcm"}, self.loop) + await self.msg_admin.aiowrite( + "admin", "ping", + {"from": "lcm", "to": "lcm", "worker_id": self.worker_id, "version": lcm_version}, + self.loop) # time between pings are low when it is not received and at starting wait_time = self.ping_interval_boot if not kafka_has_received else self.ping_interval_pace if not self.pings_not_received: @@ -193,196 +229,213 @@ class Lcm: raise consecutive_errors += 1 self.logger.error("Task kafka_read retrying after Exception {}".format(e)) - wait_time = 1 if not first_start else 5 + wait_time = 2 if not first_start else 5 await asyncio.sleep(wait_time, loop=self.loop) - async def kafka_read(self): - self.logger.debug("Task kafka_read Enter") + def kafka_read_callback(self, topic, command, params): order_id = 1 + + if topic != "admin" and command != "ping": + self.logger.debug("Task kafka_read receives {} {}: {}".format(topic, command, params)) + self.consecutive_errors = 0 + self.first_start = False + order_id += 1 + if command == "exit": + raise LcmExceptionExit + elif command.startswith("#"): + return + elif command == "echo": + # just for test + print(params) + sys.stdout.flush() + return + elif command == "test": + asyncio.Task(self.test(params), loop=self.loop) + return + + if topic == "admin": + if command == "ping" and params["to"] == "lcm" and params["from"] == "lcm": + if params.get("worker_id") != self.worker_id: + return + self.pings_not_received = 0 + try: + with open(health_check_file, "w") as f: + f.write(str(time())) + except Exception as e: + self.logger.error("Cannot write into '{}' for healthcheck: {}".format(health_check_file, e)) + return + elif topic == "ns": + if command == "instantiate": + # self.logger.debug("Deploying NS {}".format(nsr_id)) + nslcmop = params + nslcmop_id = nslcmop["_id"] + nsr_id = nslcmop["nsInstanceId"] + task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id)) + self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task) + return + elif command == "terminate": + # self.logger.debug("Deleting NS {}".format(nsr_id)) + nslcmop = params + nslcmop_id = nslcmop["_id"] + nsr_id = nslcmop["nsInstanceId"] + self.lcm_tasks.cancel(topic, nsr_id) + task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id)) + self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_terminate", task) + return + elif command == "action": + # self.logger.debug("Update NS {}".format(nsr_id)) + nslcmop = params + nslcmop_id = nslcmop["_id"] + nsr_id = nslcmop["nsInstanceId"] + task = asyncio.ensure_future(self.ns.action(nsr_id, nslcmop_id)) + self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_action", task) + return + elif command == "scale": + # self.logger.debug("Update NS {}".format(nsr_id)) + nslcmop = params + nslcmop_id = nslcmop["_id"] + nsr_id = nslcmop["nsInstanceId"] + task = asyncio.ensure_future(self.ns.scale(nsr_id, nslcmop_id)) + self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_scale", task) + return + elif command == "show": + nsr_id = params + try: + db_nsr = self.db.get_one("nsrs", {"_id": nsr_id}) + print("nsr:\n _id={}\n operational-status: {}\n config-status: {}" + "\n detailed-status: {}\n deploy: {}\n tasks: {}" + "".format(nsr_id, db_nsr["operational-status"], db_nsr["config-status"], + db_nsr["detailed-status"], + db_nsr["_admin"]["deployed"], self.lcm_ns_tasks.get(nsr_id))) + except Exception as e: + print("nsr {} not found: {}".format(nsr_id, e)) + sys.stdout.flush() + return + elif command == "deleted": + return # TODO cleaning of task just in case should be done + elif command in ("terminated", "instantiated", "scaled", "actioned"): # "scaled-cooldown-time" + return + elif topic == "nsi": # netslice LCM processes (instantiate, terminate, etc) + if command == "instantiate": + # self.logger.debug("Instantiating Network Slice {}".format(nsilcmop["netsliceInstanceId"])) + nsilcmop = params + nsilcmop_id = nsilcmop["_id"] # slice operation id + nsir_id = nsilcmop["netsliceInstanceId"] # slice record id + task = asyncio.ensure_future(self.netslice.instantiate(nsir_id, nsilcmop_id)) + self.lcm_tasks.register("nsi", nsir_id, nsilcmop_id, "nsi_instantiate", task) + return + elif command == "terminate": + # self.logger.debug("Terminating Network Slice NS {}".format(nsilcmop["netsliceInstanceId"])) + nsilcmop = params + nsilcmop_id = nsilcmop["_id"] # slice operation id + nsir_id = nsilcmop["netsliceInstanceId"] # slice record id + self.lcm_tasks.cancel(topic, nsir_id) + task = asyncio.ensure_future(self.netslice.terminate(nsir_id, nsilcmop_id)) + self.lcm_tasks.register("nsi", nsir_id, nsilcmop_id, "nsi_terminate", task) + return + elif command == "show": + nsir_id = params + try: + db_nsir = self.db.get_one("nsirs", {"_id": nsir_id}) + print("nsir:\n _id={}\n operational-status: {}\n config-status: {}" + "\n detailed-status: {}\n deploy: {}\n tasks: {}" + "".format(nsir_id, db_nsir["operational-status"], db_nsir["config-status"], + db_nsir["detailed-status"], + db_nsir["_admin"]["deployed"], self.lcm_netslice_tasks.get(nsir_id))) + except Exception as e: + print("nsir {} not found: {}".format(nsir_id, e)) + sys.stdout.flush() + return + elif command == "deleted": + return # TODO cleaning of task just in case should be done + elif command in ("terminated", "instantiated", "scaled", "actioned"): # "scaled-cooldown-time" + return + elif topic == "vim_account": + vim_id = params["_id"] + if command == "create": + task = asyncio.ensure_future(self.vim.create(params, order_id)) + self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_create", task) + return + elif command == "delete": + self.lcm_tasks.cancel(topic, vim_id) + task = asyncio.ensure_future(self.vim.delete(vim_id, order_id)) + self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_delete", task) + return + elif command == "show": + print("not implemented show with vim_account") + sys.stdout.flush() + return + elif command == "edit": + task = asyncio.ensure_future(self.vim.edit(params, order_id)) + self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_edit", task) + return + elif topic == "wim_account": + wim_id = params["_id"] + if command == "create": + task = asyncio.ensure_future(self.wim.create(params, order_id)) + self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_create", task) + return + elif command == "delete": + self.lcm_tasks.cancel(topic, wim_id) + task = asyncio.ensure_future(self.wim.delete(wim_id, order_id)) + self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_delete", task) + return + elif command == "show": + print("not implemented show with wim_account") + sys.stdout.flush() + return + elif command == "edit": + task = asyncio.ensure_future(self.wim.edit(params, order_id)) + self.lcm_tasks.register("wim_account", wim_id, order_id, "wim_edit", task) + return + elif topic == "sdn": + _sdn_id = params["_id"] + if command == "create": + task = asyncio.ensure_future(self.sdn.create(params, order_id)) + self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_create", task) + return + elif command == "delete": + self.lcm_tasks.cancel(topic, _sdn_id) + task = asyncio.ensure_future(self.sdn.delete(_sdn_id, order_id)) + self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_delete", task) + return + elif command == "edit": + task = asyncio.ensure_future(self.sdn.edit(params, order_id)) + self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_edit", task) + return + self.logger.critical("unknown topic {} and command '{}'".format(topic, command)) + + async def kafka_read(self): + self.logger.debug("Task kafka_read Enter with worker_id={}".format(self.worker_id)) # future = asyncio.Future() - consecutive_errors = 0 - first_start = True - while consecutive_errors < 10: + self.consecutive_errors = 0 + self.first_start = True + while self.consecutive_errors < 10: try: - topics = ("admin", "ns", "vim_account", "sdn", "nsi") - topic, command, params = await self.msg.aioread(topics, self.loop) - if topic != "admin" and command != "ping": - self.logger.debug("Task kafka_read receives {} {}: {}".format(topic, command, params)) - consecutive_errors = 0 - first_start = False - order_id += 1 - if command == "exit": - print("Bye!") - break - elif command.startswith("#"): - continue - elif command == "echo": - # just for test - print(params) - sys.stdout.flush() - continue - elif command == "test": - asyncio.Task(self.test(params), loop=self.loop) - continue - - if topic == "admin": - if command == "ping" and params["to"] == "lcm" and params["from"] == "lcm": - self.pings_not_received = 0 - continue - elif topic == "ns": - if command == "instantiate": - # self.logger.debug("Deploying NS {}".format(nsr_id)) - nslcmop = params - nslcmop_id = nslcmop["_id"] - nsr_id = nslcmop["nsInstanceId"] - task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id)) - self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task) - continue - elif command == "terminate": - # self.logger.debug("Deleting NS {}".format(nsr_id)) - nslcmop = params - nslcmop_id = nslcmop["_id"] - nsr_id = nslcmop["nsInstanceId"] - self.lcm_tasks.cancel(topic, nsr_id) - task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id)) - self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_terminate", task) - continue - elif command == "action": - # self.logger.debug("Update NS {}".format(nsr_id)) - nslcmop = params - nslcmop_id = nslcmop["_id"] - nsr_id = nslcmop["nsInstanceId"] - task = asyncio.ensure_future(self.ns.action(nsr_id, nslcmop_id)) - self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_action", task) - continue - elif command == "scale": - # self.logger.debug("Update NS {}".format(nsr_id)) - nslcmop = params - nslcmop_id = nslcmop["_id"] - nsr_id = nslcmop["nsInstanceId"] - task = asyncio.ensure_future(self.ns.scale(nsr_id, nslcmop_id)) - self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_scale", task) - continue - elif command == "show": - try: - db_nsr = self.db.get_one("nsrs", {"_id": nsr_id}) - print("nsr:\n _id={}\n operational-status: {}\n config-status: {}" - "\n detailed-status: {}\n deploy: {}\n tasks: {}" - "".format(nsr_id, db_nsr["operational-status"], db_nsr["config-status"], - db_nsr["detailed-status"], - db_nsr["_admin"]["deployed"], self.lcm_ns_tasks.get(nsr_id))) - except Exception as e: - print("nsr {} not found: {}".format(nsr_id, e)) - sys.stdout.flush() - continue - elif command == "deleted": - continue # TODO cleaning of task just in case should be done - elif command in ("terminated", "instantiated", "scaled", "actioned"): # "scaled-cooldown-time" - continue - elif topic == "nsi": # netslice LCM processes (instantiate, terminate, etc) - if command == "instantiate": - # self.logger.debug("Instantiating Network Slice {}".format(nsilcmop["netsliceInstanceId"])) - nsilcmop = params - nsilcmop_id = nsilcmop["_id"] # slice operation id - nsir_id = nsilcmop["netsliceInstanceId"] # slice record id - task = asyncio.ensure_future(self.netslice.instantiate(nsir_id, nsilcmop_id)) - self.lcm_tasks.register("nsi", nsir_id, nsilcmop_id, "nsi_instantiate", task) - continue - elif command == "terminate": - # self.logger.debug("Terminating Network Slice NS {}".format(nsilcmop["netsliceInstanceId"])) - nsilcmop = params - nsilcmop_id = nsilcmop["_id"] # slice operation id - nsir_id = nsilcmop["netsliceInstanceId"] # slice record id - self.lcm_tasks.cancel(topic, nsir_id) - task = asyncio.ensure_future(self.netslice.terminate(nsir_id, nsilcmop_id)) - self.lcm_tasks.register("nsi", nsir_id, nsilcmop_id, "nsi_terminate", task) - continue - elif command == "show": - try: - db_nsir = self.db.get_one("nsirs", {"_id": nsir_id}) - print("nsir:\n _id={}\n operational-status: {}\n config-status: {}" - "\n detailed-status: {}\n deploy: {}\n tasks: {}" - "".format(nsir_id, db_nsir["operational-status"], db_nsir["config-status"], - db_nsir["detailed-status"], - db_nsir["_admin"]["deployed"], self.lcm_netslice_tasks.get(nsir_id))) - except Exception as e: - print("nsir {} not found: {}".format(nsir_id, e)) - sys.stdout.flush() - continue - elif command == "deleted": - continue # TODO cleaning of task just in case should be done - elif command in ("terminated", "instantiated", "scaled", "actioned"): # "scaled-cooldown-time" - continue - elif topic == "vim_account": - vim_id = params["_id"] - if command == "create": - task = asyncio.ensure_future(self.vim.create(params, order_id)) - self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_create", task) - continue - elif command == "delete": - self.lcm_tasks.cancel(topic, vim_id) - task = asyncio.ensure_future(self.vim.delete(vim_id, order_id)) - self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_delete", task) - continue - elif command == "show": - print("not implemented show with vim_account") - sys.stdout.flush() - continue - elif command == "edit": - task = asyncio.ensure_future(self.vim.edit(params, order_id)) - self.lcm_tasks.register("vim_account", vim_id, order_id, "vim_edit", task) - continue - elif topic == "sdn": - _sdn_id = params["_id"] - if command == "create": - task = asyncio.ensure_future(self.sdn.create(params, order_id)) - self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_create", task) - continue - elif command == "delete": - self.lcm_tasks.cancel(topic, _sdn_id) - task = asyncio.ensure_future(self.sdn.delete(_sdn_id, order_id)) - self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_delete", task) - continue - elif command == "edit": - task = asyncio.ensure_future(self.sdn.edit(params, order_id)) - self.lcm_tasks.register("sdn", _sdn_id, order_id, "sdn_edit", task) - continue - self.logger.critical("unknown topic {} and command '{}'".format(topic, command)) + topics = ("ns", "vim_account", "wim_account", "sdn", "nsi") + topics_admin = ("admin", ) + await asyncio.gather( + self.msg.aioread(topics, self.loop, self.kafka_read_callback), + self.msg_admin.aioread(topics_admin, self.loop, self.kafka_read_callback, group_id=False) + ) + + except LcmExceptionExit: + self.logger.debug("Bye!") + break except Exception as e: # if not first_start is the first time after starting. So leave more time and wait # to allow kafka starts - if consecutive_errors == 8 if not first_start else 30: + if self.consecutive_errors == 8 if not self.first_start else 30: self.logger.error("Task kafka_read task exit error too many errors. Exception: {}".format(e)) raise - consecutive_errors += 1 + self.consecutive_errors += 1 self.logger.error("Task kafka_read retrying after Exception {}".format(e)) - wait_time = 2 if not first_start else 5 + wait_time = 2 if not self.first_start else 5 await asyncio.sleep(wait_time, loop=self.loop) # self.logger.debug("Task kafka_read terminating") self.logger.debug("Task kafka_read exit") - def health_check(self): - - global exit_code - task = None - exit_code = 1 - - def health_check_callback(topic, command, params): - global exit_code - print("receiving callback {} {} {}".format(topic, command, params)) - if topic == "admin" and command == "ping" and params["to"] == "lcm" and params["from"] == "lcm": - # print("received LCM ping") - exit_code = 0 - task.cancel() - - try: - task = asyncio.ensure_future(self.msg.aioread(("admin",), self.loop, health_check_callback)) - self.loop.run_until_complete(task) - except Exception: - pass - exit(exit_code) - def start(self): # check RO version @@ -408,6 +461,8 @@ class Lcm: self.db.db_disconnect() if self.msg: self.msg.disconnect() + if self.msg_admin: + self.msg_admin.disconnect() if self.fs: self.fs.fs_disconnect() @@ -443,6 +498,26 @@ class Lcm: self.logger.critical("At config file '{}': {}".format(config_file, e)) exit(1) + @staticmethod + def get_process_id(): + """ + Obtain a unique ID for this process. If running from inside docker, it will get docker ID. If not it + will provide a random one + :return: Obtained ID + """ + # Try getting docker id. If fails, get pid + try: + with open("/proc/self/cgroup", "r") as f: + text_id_ = f.readline() + _, _, text_id = text_id_.rpartition("/") + text_id = text_id.replace('\n', '')[:12] + if text_id: + return text_id + except Exception: + pass + # Return a random id + return ''.join(random_choice("0123456789abcdef") for _ in range(12)) + def usage(): print("""Usage: {} [options] @@ -454,13 +529,29 @@ def usage(): # --log-socket-port PORT: send logs using this port (default: 9022)") +def health_check(): + retry = 2 + while retry: + retry -= 1 + try: + with open(health_check_file, "r") as f: + last_received_ping = f.read() + + if time() - float(last_received_ping) < Lcm.ping_interval_pace + 10: + exit(0) + except Exception: + pass + if retry: + sleep(6) + exit(1) + + if __name__ == '__main__': try: # load parameters and configuration opts, args = getopt.getopt(sys.argv[1:], "hc:", ["config=", "help", "health-check"]) # TODO add "log-socket-host=", "log-socket-port=", "log-file=" config_file = None - health_check = None for o, a in opts: if o in ("-h", "--help"): usage() @@ -468,7 +559,7 @@ if __name__ == '__main__': elif o in ("-c", "--config"): config_file = a elif o == "--health-check": - health_check = True + health_check() # elif o == "--log-socket-port": # log_socket_port = a # elif o == "--log-socket-host": @@ -489,10 +580,7 @@ if __name__ == '__main__': print("No configuration file 'lcm.cfg' found neither at local folder nor at /etc/osm/", file=sys.stderr) exit(1) lcm = Lcm(config_file) - if health_check: - lcm.health_check() - else: - lcm.start() + lcm.start() except (LcmException, getopt.GetoptError) as e: print(str(e), file=sys.stderr) # usage()